Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'bugfix/confgen_gen_source_from_build' into 'master'

Browse source 
Generate source files for kconfiglib from the build system

See merge request espressif/esp-idf!6433
This commit is contained in:
Angus Gratton 2019-11-19 14:19:49 +08:00
parent 6f633b2d5b 84d5cc1c17
commit af945096b2
11 changed files with 135 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
docs/conf_common.py
View file

@ -103,6 +103,15 @@ sdkconfig_renames = [r for r in sdkconfig_renames if "esp32s2beta" not in r]
kconfigs_source_path = '{}/inc/kconfigs_source.in'.format(builddir)
kconfig_projbuilds_source_path = '{}/inc/kconfig_projbuilds_source.in'.format(builddir)
prepare_kconfig_files_args = [sys.executable,
"../../tools/kconfig_new/prepare_kconfig_files.py",
"--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)),
"--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)),
"--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path),
"--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path),
]
subprocess.check_call(prepare_kconfig_files_args)
confgen_args = [sys.executable,
"../../tools/kconfig_new/confgen.py",
"--kconfig", "../../Kconfig",

4
make/ldgen.mk
View file

@ -9,7 +9,7 @@ define ldgen_process_template
$(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk
printf "$(foreach info,$(LDGEN_LIBRARIES),$(subst \,/,$(shell cygpath -m $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen_libraries
$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries
$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries | prepare_kconfig_files
@echo 'Generating $(notdir $(2))'
$(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \
--input $(1) \
@ -30,7 +30,7 @@ define ldgen_process_template
$(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk
printf "$(foreach library,$(LDGEN_LIBRARIES),$(library)\n)" > $(BUILD_DIR_BASE)/ldgen_libraries
$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries
$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries | prepare_kconfig_files
@echo 'Generating $(notdir $(2))'
$(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \
--input $(1) \

14
make/project_config.mk
View file

@ -70,6 +70,14 @@ SDKCONFIG_DEFAULTS_FILES := $(shell cygpath -m $(SDKCONFIG_DEFAULTS_FILES))
endif
DEFAULTS_ARG := $(foreach f,$(SDKCONFIG_DEFAULTS_FILES),--defaults $(f))
prepare_kconfig_files:
mkdir -p $(BUILD_DIR_BASE)
$(PYTHON) $(IDF_PATH)/tools/kconfig_new/prepare_kconfig_files.py \
--env "COMPONENT_KCONFIGS=$(strip $(COMPONENT_KCONFIGS))" \
--env "COMPONENT_KCONFIGS_PROJBUILD=$(strip $(COMPONENT_KCONFIGS_PROJBUILD))" \
--env "COMPONENT_KCONFIGS_SOURCE_FILE=$(COMPONENT_KCONFIGS_SOURCE_FILE)" \
--env "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE=$(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE)"
# macro for running confgen.py
define RunConfGen
mkdir -p $(BUILD_DIR_BASE)/include/config
@ -118,7 +126,7 @@ ifndef MAKE_RESTARTS
# depend on any prerequisite that may cause a make restart as part of
# the prerequisite's own recipe.
menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf | check_python_dependencies
menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf | check_python_dependencies prepare_kconfig_files
$(summary) MENUCONFIG
ifdef BATCH_BUILD
@echo "Can't run interactive configuration inside non-interactive build process."
@ -135,13 +143,13 @@ else
endif
# defconfig creates a default config, based on SDKCONFIG_DEFAULTS if present
defconfig: | check_python_dependencies
defconfig: | check_python_dependencies prepare_kconfig_files
$(summary) DEFCONFIG
$(call RunConfGen)
# if neither defconfig or menuconfig are requested, use the GENCONFIG rule to
# ensure generated config files are up to date
$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | check_python_dependencies $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig)
$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | check_python_dependencies prepare_kconfig_files $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig)
$(summary) GENCONFIG
$(call RunConfGen)
touch $(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h # ensure newer than sdkconfig

10
tools/cmake/kconfig.cmake
View file

@ -175,6 +175,10 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
idf_build_get_property(root_sdkconfig_rename __ROOT_SDKCONFIG_RENAME)
idf_build_get_property(python PYTHON)
set(prepare_kconfig_files_command
${python} ${idf_path}/tools/kconfig_new/prepare_kconfig_files.py
--env-file ${config_env_path})
set(confgen_basecommand
${python} ${idf_path}/tools/kconfig_new/confgen.py
--kconfig ${root_kconfig}
@ -195,6 +199,8 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
idf_build_get_property(output_sdkconfig __OUTPUT_SDKCONFIG)
if(output_sdkconfig)
execute_process(
COMMAND ${prepare_kconfig_files_command})
execute_process(
COMMAND ${confgen_basecommand}
--output header ${sdkconfig_header}
@ -204,6 +210,8 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
--output config ${sdkconfig}
RESULT_VARIABLE config_result)
else()
execute_process(
COMMAND ${prepare_kconfig_files_command})
execute_process(
COMMAND ${confgen_basecommand}
--output header ${sdkconfig_header}
@ -254,6 +262,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
add_custom_target(menuconfig
${menuconfig_depends}
# create any missing config file, with defaults if necessary
COMMAND ${prepare_kconfig_files_command}
COMMAND ${confgen_basecommand} --env "IDF_TARGET=${idf_target}" --output config ${sdkconfig}
COMMAND ${CMAKE_COMMAND} -E env
"COMPONENT_KCONFIGS_SOURCE_FILE=${kconfigs_path}"
@ -273,6 +282,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
# Custom target to run confserver.py from the build tool
add_custom_target(confserver
COMMAND ${prepare_kconfig_files_command}
COMMAND ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py
--env-file ${config_env_path}
--kconfig ${IDF_PATH}/Kconfig

22
tools/kconfig_new/confgen.py
View file

@ -175,27 +175,6 @@ class DeprecatedOptions(object):
f_o.write('#define {}{} {}{}\n'.format(self.config_prefix, dep_opt, self.config_prefix, new_opt))
def prepare_source_files():
"""
Prepares source files which are sourced from the main Kconfig because upstream kconfiglib doesn't support sourcing
a file list.
"""
def _dequote(var):
return var[1:-1] if len(var) > 0 and (var[0], var[-1]) == ('"',) * 2 else var
def _write_source_file(config_var, config_file):
with open(config_file, "w") as f:
f.write('\n'.join(['source "{}"'.format(path) for path in _dequote(config_var).split()]))
try:
_write_source_file(os.environ['COMPONENT_KCONFIGS'], os.environ['COMPONENT_KCONFIGS_SOURCE_FILE'])
_write_source_file(os.environ['COMPONENT_KCONFIGS_PROJBUILD'], os.environ['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE'])
except KeyError as e:
print('Error:', e, 'is not defined!')
raise
def dict_enc_for_env(dic, encoding=sys.getfilesystemencoding() or 'utf-8'):
"""
This function can be deleted after dropping support for Python 2.
@ -266,7 +245,6 @@ def main():
env = json.load(args.env_file)
os.environ.update(dict_enc_for_env(env))
prepare_source_files()
config = kconfiglib.Kconfig(args.kconfig)
config.warn_assign_redun = False
config.warn_assign_override = False

1
tools/kconfig_new/confserver.py
View file

@ -75,7 +75,6 @@ def main():
def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCOL_VERSION):
confgen.prepare_source_files()
config = kconfiglib.Kconfig(kconfig)
sdkconfig_renames = [sdkconfig_rename] if sdkconfig_rename else []
sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()

94
tools/kconfig_new/prepare_kconfig_files.py Normal file
View file

@ -0,0 +1,94 @@
#!/usr/bin/env python
#
# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
from io import open
import argparse
import json
import sys
def _prepare_source_files(env_dict):
"""
Prepares source files which are sourced from the main Kconfig because upstream kconfiglib doesn't support sourcing
a file list. The inputs are the same environment variables which are used by kconfiglib:
- COMPONENT_KCONFIGS,
- COMPONENT_KCONFIGS_SOURCE_FILE,
- COMPONENT_KCONFIGS_PROJBUILD,
- COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE.
The outputs are written into files pointed by the value of
- COMPONENT_KCONFIGS_SOURCE_FILE,
- COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE,
After running this function, COMPONENT_KCONFIGS_SOURCE_FILE and COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE will
contain a list of source statements based on the content of COMPONENT_KCONFIGS and COMPONENT_KCONFIGS_PROJBUILD,
respectively. For example, if COMPONENT_KCONFIGS="var1 var2 var3" and
COMPONENT_KCONFIGS_SOURCE_FILE="/path/file.txt" then the content of file /path/file.txt will be:
source "var1"
source "var2"
source "var3"
"""
def _dequote(var):
return var[1:-1] if len(var) > 0 and (var[0], var[-1]) == ('"',) * 2 else var
def _write_source_file(config_var, config_file):
new_content = '\n'.join(['source "{}"'.format(path) for path in _dequote(config_var).split()])
try:
with open(config_file, 'r', encoding='utf-8') as f:
old_content = f.read()
except Exception:
# File doesn't exist or other issue
old_content = ''
if new_content != old_content:
# write or rewrite file only if it is necessary
with open(config_file, 'w', encoding='utf-8') as f:
f.write(new_content)
try:
_write_source_file(env_dict['COMPONENT_KCONFIGS'], env_dict['COMPONENT_KCONFIGS_SOURCE_FILE'])
_write_source_file(env_dict['COMPONENT_KCONFIGS_PROJBUILD'], env_dict['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE'])
except KeyError as e:
print('Error:', e, 'is not defined!')
sys.exit(1)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Kconfig Source File Generator')
parser.add_argument('--env', action='append', default=[],
help='Environment value', metavar='NAME=VAL')
parser.add_argument('--env-file', type=argparse.FileType('r'),
help='Optional file to load environment variables from. Contents '
'should be a JSON object where each key/value pair is a variable.')
args = parser.parse_args()
try:
env = dict([(name, value) for (name, value) in (e.split("=", 1) for e in args.env)])
except ValueError:
print("--env arguments must each contain =.")
sys.exit(1)
if args.env_file is not None:
env.update(json.load(args.env_file))
_prepare_source_files(env)

3
tools/kconfig_new/test/test_confserver.py
View file

@ -66,6 +66,9 @@ def main():
cmdline = re.sub(r' +', ' ', cmdline)
# prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and
# COMPONENT_KCONFIGS_PROJBUILD are empty
print("Running: %s" % cmdline)
p = pexpect.spawn(cmdline, timeout=30, logfile=args.logfile, echo=False, use_poll=True, maxread=1)

3
tools/ldgen/sdkconfig.py
View file

@ -26,8 +26,6 @@ except Exception:
sys.path.insert(0, kconfig_new_dir)
import kconfiglib
import confgen
class SDKConfig:
"""
@ -49,7 +47,6 @@ class SDKConfig:
OPERATOR = oneOf(["=", "!=", ">", "<", "<=", ">="])
def __init__(self, kconfig_file, sdkconfig_file):
confgen.prepare_source_files()
self.config = kconfiglib.Kconfig(kconfig_file)
self.config.load_config(sdkconfig_file)

3
tools/ldgen/test/test_fragments.py
View file

@ -69,6 +69,9 @@ class FragmentTest(unittest.TestCase):
os.environ['COMPONENT_KCONFIGS'] = ''
os.environ['COMPONENT_KCONFIGS_PROJBUILD'] = ''
# prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and
# COMPONENT_KCONFIGS_PROJBUILD are empty
self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
def tearDown(self):

3
tools/ldgen/test/test_generation.py
View file

@ -56,6 +56,9 @@ class GenerationModelTest(unittest.TestCase):
os.environ['COMPONENT_KCONFIGS'] = ''
os.environ['COMPONENT_KCONFIGS_PROJBUILD'] = ''
# prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and
# COMPONENT_KCONFIGS_PROJBUILD are empty
self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
with open("data/sample.lf") as fragment_file_obj: