Merge branch 'bugfix/confserver_temporaryfile_windows_v4.0' into 'release/v4.0'

confserver: Fix NamedTemporaryFile use on Windows (v4.0)

See merge request espressif/esp-idf!5818

docs/conf_common.py

@@ -93,13 +93,16 @@ temp_sdkconfig_path = '{}/sdkconfig.tmp'.format(builddir)
 
 kconfigs = find_component_files("../../components", "Kconfig")
 kconfig_projbuilds = find_component_files("../../components", "Kconfig.projbuild")
+sdkconfig_renames = find_component_files("../../components", "sdkconfig.rename")
 
 confgen_args = [sys.executable,
                 "../../tools/kconfig_new/confgen.py",
                 "--kconfig", "../../Kconfig",
+                "--sdkconfig-rename", "../../sdkconfig.rename",
                 "--config", temp_sdkconfig_path,
                 "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)),
                 "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)),
+                "--env", "COMPONENT_SDKCONFIG_RENAMES={}".format(" ".join(sdkconfig_renames)),
                 "--env", "IDF_PATH={}".format(idf_path),
                 "--output", "docs", kconfig_inc_path + '.in'
                 ]

make/project_config.mk

@@ -3,6 +3,7 @@
 #Find all Kconfig files for all components
 COMPONENT_KCONFIGS := $(foreach component,$(COMPONENT_PATHS),$(wildcard $(component)/Kconfig))
 COMPONENT_KCONFIGS_PROJBUILD := $(foreach component,$(COMPONENT_PATHS),$(wildcard $(component)/Kconfig.projbuild))
+COMPONENT_SDKCONFIG_RENAMES := $(foreach component,$(COMPONENT_PATHS),$(wildcard $(component)/sdkconfig.rename))
 
 ifeq ($(OS),Windows_NT)
 # kconfiglib requires Windows-style paths for kconfig files
@@ -17,6 +18,8 @@ KCONFIG_TOOL_DIR=$(IDF_PATH)/tools/kconfig
 # unless it's overriden (happens for bootloader)
 SDKCONFIG ?= $(PROJECT_PATH)/sdkconfig
 
+SDKCONFIG_RENAME ?= $(IDF_PATH)/sdkconfig.rename
+
 # SDKCONFIG_DEFAULTS is an optional file containing default
 # overrides (usually used for esp-idf examples)
 SDKCONFIG_DEFAULTS ?= $(PROJECT_PATH)/sdkconfig.defaults
@@ -48,8 +51,10 @@ define RunConfGen
 	$(PYTHON) $(IDF_PATH)/tools/kconfig_new/confgen.py \
 		--kconfig $(IDF_PATH)/Kconfig \
 		--config $(SDKCONFIG) \
+		--sdkconfig-rename $(SDKCONFIG_RENAME) \
 		--env "COMPONENT_KCONFIGS=$(strip $(COMPONENT_KCONFIGS))" \
 		--env "COMPONENT_KCONFIGS_PROJBUILD=$(strip $(COMPONENT_KCONFIGS_PROJBUILD))" \
+		--env "COMPONENT_SDKCONFIG_RENAMES=$(strip $(COMPONENT_SDKCONFIG_RENAMES))" \
 		--env "IDF_CMAKE=n" \
 		--output config ${SDKCONFIG} \
 		--output makefile $(SDKCONFIG_MAKEFILE) \

tools/cmake/kconfig.cmake

@@ -72,6 +72,7 @@ function(__kconfig_init)
 
     idf_build_get_property(idf_path IDF_PATH)
     idf_build_set_property(__ROOT_KCONFIG ${idf_path}/Kconfig)
+    idf_build_set_property(__ROOT_SDKCONFIG_RENAME ${idf_path}/sdkconfig.rename)
     idf_build_set_property(__OUTPUT_SDKCONFIG 1)
 endfunction()
 
@@ -86,6 +87,8 @@ function(__kconfig_component_init component_target)
     __component_set_property(${component_target} KCONFIG "${kconfig}")
     file(GLOB kconfig "${component_dir}/Kconfig.projbuild")
     __component_set_property(${component_target} KCONFIG_PROJBUILD "${kconfig}")
+    file(GLOB sdkconfig_rename "${component_dir}/sdkconfig.rename")
+    __component_set_property(${component_target} SDKCONFIG_RENAME "${sdkconfig_rename}")
 endfunction()
 
 #
@@ -100,12 +103,16 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
         if(component_target IN_LIST build_component_targets)
             __component_get_property(kconfig ${component_target} KCONFIG)
             __component_get_property(kconfig_projbuild ${component_target} KCONFIG_PROJBUILD)
+            __component_get_property(sdkconfig_rename ${component_target} SDKCONFIG_RENAME)
             if(kconfig)
                 list(APPEND kconfigs ${kconfig})
             endif()
             if(kconfig_projbuild)
                 list(APPEND kconfig_projbuilds ${kconfig_projbuild})
             endif()
+            if(sdkconfig_rename)
+                list(APPEND sdkconfig_renames ${sdkconfig_rename})
+            endif()
         endif()
     endforeach()
 
@@ -118,6 +125,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
 
     string(REPLACE ";" " " kconfigs "${kconfigs}")
     string(REPLACE ";" " " kconfig_projbuilds "${kconfig_projbuilds}")
+    string(REPLACE ";" " " sdkconfig_renames "${sdkconfig_renames}")
 
     # Place config-related environment arguments into config.env file
     # to work around command line length limits for execute_process
@@ -135,11 +143,13 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
     endif()
 
     idf_build_get_property(root_kconfig __ROOT_KCONFIG)
+    idf_build_get_property(root_sdkconfig_rename __ROOT_SDKCONFIG_RENAME)
     idf_build_get_property(python PYTHON)
 
     set(confgen_basecommand
         ${python} ${idf_path}/tools/kconfig_new/confgen.py
         --kconfig ${root_kconfig}
+        --sdkconfig-rename ${root_sdkconfig_rename}
         --config ${sdkconfig}
         ${defaults_arg}
         --env-file ${config_env_path})
@@ -231,6 +241,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
         COMMAND ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py
         --env-file ${config_env_path}
         --kconfig ${IDF_PATH}/Kconfig
+        --sdkconfig-rename ${root_sdkconfig_rename}
         --config ${sdkconfig}
         VERBATIM
         USES_TERMINAL)

tools/kconfig_new/confgen.py

@@ -22,7 +22,6 @@
 # limitations under the License.
 from __future__ import print_function
 import argparse
-import fnmatch
 import json
 import os
 import os.path
@@ -46,15 +45,15 @@ class DeprecatedOptions(object):
     _RE_DEP_OP_BEGIN = re.compile(_DEP_OP_BEGIN)
     _RE_DEP_OP_END = re.compile(_DEP_OP_END)
 
-    def __init__(self, config_prefix, path_rename_files, ignore_dirs=()):
+    def __init__(self, config_prefix, path_rename_files=[]):
         self.config_prefix = config_prefix
         # r_dic maps deprecated options to new options; rev_r_dic maps in the opposite direction
-        self.r_dic, self.rev_r_dic = self._parse_replacements(path_rename_files, ignore_dirs)
+        self.r_dic, self.rev_r_dic = self._parse_replacements(path_rename_files)
 
         # note the '=' at the end of regex for not getting partial match of configs
         self._RE_CONFIG = re.compile(r'{}(\w+)='.format(self.config_prefix))
 
-    def _parse_replacements(self, repl_dir, ignore_dirs):
+    def _parse_replacements(self, repl_paths):
         rep_dic = {}
         rev_rep_dic = {}
 
@@ -64,31 +63,24 @@ class DeprecatedOptions(object):
             raise RuntimeError('Error in {} (line {}): Config {} is not prefixed with {}'
                                ''.format(rep_path, line_number, string, self.config_prefix))
 
-        for root, dirnames, filenames in os.walk(repl_dir):
+        for rep_path in repl_paths:
-            for filename in fnmatch.filter(filenames, self._REN_FILE):
+            with open(rep_path) as f_rep:
-                rep_path = os.path.join(root, filename)
+                for line_number, line in enumerate(f_rep, start=1):
+                    sp_line = line.split()
+                    if len(sp_line) == 0 or sp_line[0].startswith('#'):
+                        # empty line or comment
+                        continue
+                    if len(sp_line) != 2 or not all(x.startswith(self.config_prefix) for x in sp_line):
+                        raise RuntimeError('Syntax error in {} (line {})'.format(rep_path, line_number))
+                    if sp_line[0] in rep_dic:
+                        raise RuntimeError('Error in {} (line {}): Replacement {} exist for {} and new '
+                                           'replacement {} is defined'.format(rep_path, line_number,
+                                                                              rep_dic[sp_line[0]], sp_line[0],
+                                                                              sp_line[1]))
 
-                if rep_path.startswith(ignore_dirs):
+                    (dep_opt, new_opt) = (remove_config_prefix(x) for x in sp_line)
-                    print('Ignoring: {}'.format(rep_path))
+                    rep_dic[dep_opt] = new_opt
-                    continue
+                    rev_rep_dic[new_opt] = dep_opt
-
-                with open(rep_path) as f_rep:
-                    for line_number, line in enumerate(f_rep, start=1):
-                        sp_line = line.split()
-                        if len(sp_line) == 0 or sp_line[0].startswith('#'):
-                            # empty line or comment
-                            continue
-                        if len(sp_line) != 2 or not all(x.startswith(self.config_prefix) for x in sp_line):
-                            raise RuntimeError('Syntax error in {} (line {})'.format(rep_path, line_number))
-                        if sp_line[0] in rep_dic:
-                            raise RuntimeError('Error in {} (line {}): Replacement {} exist for {} and new '
-                                               'replacement {} is defined'.format(rep_path, line_number,
-                                                                                  rep_dic[sp_line[0]], sp_line[0],
-                                                                                  sp_line[1]))
-
-                        (dep_opt, new_opt) = (remove_config_prefix(x) for x in sp_line)
-                        rep_dic[dep_opt] = new_opt
-                        rev_rep_dic[new_opt] = dep_opt
         return rep_dic, rev_rep_dic
 
     def get_deprecated_option(self, new_option):
@@ -190,6 +182,10 @@ def main():
                         help='KConfig file with config item definitions',
                         required=True)
 
+    parser.add_argument('--sdkconfig-rename',
+                        help='File with deprecated Kconfig options',
+                        required=False)
+
     parser.add_argument('--output', nargs=2, action='append',
                         help='Write output file (format and output filename)',
                         metavar=('FORMAT', 'FILENAME'),
@@ -235,10 +231,9 @@ def main():
                 raise RuntimeError("Defaults file not found: %s" % name)
             config.load_config(name, replace=False)
 
-    # don't collect rename options from examples because those are separate projects and no need to "stay compatible"
+    sdkconfig_renames = [args.sdkconfig_rename] if args.sdkconfig_rename else []
-    # with example projects
+    sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()
-    deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=os.environ["IDF_PATH"],
+    deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames)
-                                           ignore_dirs=(os.path.join(os.environ["IDF_PATH"], 'examples')))
 
     # If config file previously exists, load it
     if args.config and os.path.exists(args.config):

tools/kconfig_new/config.env.in

@@ -1,6 +1,7 @@
 {
     "COMPONENT_KCONFIGS": "${kconfigs}",
     "COMPONENT_KCONFIGS_PROJBUILD": "${kconfig_projbuilds}",
+    "COMPONENT_SDKCONFIG_RENAMES": "${sdkconfig_renames}",
     "IDF_CMAKE": "y",
     "IDF_TARGET": "${idf_target}",
     "IDF_PATH": "${idf_path}"

tools/kconfig_new/confserver.py

@@ -29,6 +29,10 @@ def main():
                         help='KConfig file with config item definitions',
                         required=True)
 
+    parser.add_argument('--sdkconfig-rename',
+                        help='File with deprecated Kconfig options',
+                        required=False)
+
     parser.add_argument('--env', action='append', default=[],
                         help='Environment to set when evaluating the config file', metavar='NAME=VAL')
 
@@ -62,18 +66,22 @@ def main():
         env = json.load(args.env_file)
         os.environ.update(env)
 
-    run_server(args.kconfig, args.config)
+    run_server(args.kconfig, args.config, args.sdkconfig_rename)
 
 
-def run_server(kconfig, sdkconfig, default_version=MAX_PROTOCOL_VERSION):
+def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCOL_VERSION):
     config = kconfiglib.Kconfig(kconfig)
-    deprecated_options = confgen.DeprecatedOptions(config.config_prefix, path_rename_files=os.environ["IDF_PATH"])
+    sdkconfig_renames = [sdkconfig_rename] if sdkconfig_rename else []
-    with tempfile.NamedTemporaryFile(mode='w+b') as f_o:
+    sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()
+    deprecated_options = confgen.DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames)
+    f_o = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
+    try:
         with open(sdkconfig, mode='rb') as f_i:
             f_o.write(f_i.read())
-        f_o.flush()
+        f_o.close()  # need to close as DeprecatedOptions will reopen, and Windows only allows one open file
-        f_o.seek(0)
         deprecated_options.replace(sdkconfig_in=f_o.name, sdkconfig_out=sdkconfig)
+    finally:
+        os.unlink(f_o.name)
     config.load_config(sdkconfig)
 
     print("Server running, waiting for requests on stdin...", file=sys.stderr)