Update kconfiglib to upstream version and replace mconf-idf

Special thanks to @ulfalizer for the helpful suggestions regarding
kconfiglib.

"rsource" option is available for relative path includes
Closes https://github.com/espressif/esp-idf/issues/4064

.flake8

@@ -152,6 +152,7 @@ exclude =
         examples/build_system/cmake/import_lib/main/lib/tinyxml2,
     # other third-party libraries
         tools/kconfig_new/kconfiglib.py,
+        tools/kconfig_new/menuconfig.py,
     # autogenerated scripts
         components/protocomm/python/constants_pb2.py,
         components/protocomm/python/sec0_pb2.py,

Kconfig

@@ -14,20 +14,13 @@ mainmenu "Espressif IoT Development Framework Configuration"
         bool
         option env="IDF_CMAKE"
 
-
-    config IDF_TARGET_ENV
-        # A proxy to get environment variable $IDF_TARGET
-        string
-        option env="IDF_TARGET"
-
     config IDF_TARGET
         # This option records the IDF target when sdkconfig is generated the first time.
         # It is not updated if environment variable $IDF_TARGET changes later, and
         # the build system is responsible for detecting the mismatch between
         # CONFIG_IDF_TARGET and $IDF_TARGET.
         string
-        default "IDF_TARGET_NOT_SET" if IDF_TARGET_ENV=""
+        default "$(IDF_TARGET)"
-        default IDF_TARGET_ENV
 
     config IDF_TARGET_ESP32
         bool
@@ -159,7 +152,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
     endmenu # Build type
 
-    source "$COMPONENT_KCONFIGS_PROJBUILD"
+    source "$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE"
 
     menu "Compiler options"
 
@@ -326,7 +319,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
     endmenu # Compiler Options
 
     menu "Component config"
-        source "$COMPONENT_KCONFIGS"
+        source "$COMPONENT_KCONFIGS_SOURCE_FILE"
     endmenu
 
     menu "Compatibility options"

components/esp32s2beta/Kconfig

@@ -211,7 +211,7 @@ menu "ESP32S2-specific"
 
         choice SPIRAM_SPEED
             prompt "Set RAM clock speed"
-            default SPIRAM_CACHE_SPEED_40M
+            default SPIRAM_SPEED_40M
             help
                 Select the speed for the SPI RAM chip.
                 If SPI RAM is enabled, we only support three combinations of SPI speed mode we supported now:
@@ -369,7 +369,7 @@ menu "ESP32S2-specific"
     choice ESP32S2_BROWNOUT_DET_LVL_SEL
         prompt "Brownout voltage level"
         depends on ESP32S2_BROWNOUT_DET
-        default ESP32S2_BROWNOUT_DET_LVL_SEL_25
+        default ESP32S2_BROWNOUT_DET_LVL_SEL_0
         help
             The brownout detector will reset the chip when the supply voltage is approximately
             below this level. Note that there may be some variation of brownout voltage level

docs/conf_common.py

@@ -100,6 +100,9 @@ kconfigs = [k for k in kconfigs if "esp32s2beta" not in k]
 kconfig_projbuilds = [k for k in kconfig_projbuilds if "esp32s2beta" not in k]
 sdkconfig_renames = [r for r in sdkconfig_renames if "esp32s2beta" not in r]
 
+kconfigs_source_path = '{}/inc/kconfigs_source.in'.format(builddir)
+kconfig_projbuilds_source_path = '{}/inc/kconfig_projbuilds_source.in'.format(builddir)
+
 confgen_args = [sys.executable,
                 "../../tools/kconfig_new/confgen.py",
                 "--kconfig", "../../Kconfig",
@@ -108,6 +111,8 @@ confgen_args = [sys.executable,
                 "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)),
                 "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)),
                 "--env", "COMPONENT_SDKCONFIG_RENAMES={}".format(" ".join(sdkconfig_renames)),
+                "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path),
+                "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path),
                 "--env", "IDF_PATH={}".format(idf_path),
                 "--output", "docs", kconfig_inc_path + '.in'
                 ]

docs/en/COPYRIGHT.rst

@@ -4,7 +4,7 @@ Copyrights and Licenses
 Software Copyrights
 ===================
 
-All original source code in this repository is Copyright (C) 2015-2018 Espressif Systems. This source code is licensed under the Apache License 2.0 as described in the file LICENSE.
+All original source code in this repository is Copyright (C) 2015-2019 Espressif Systems. This source code is licensed under the Apache License 2.0 as described in the file LICENSE.
 
 Additional third party copyrighted code is included under the following licenses.
 
@@ -72,6 +72,10 @@ This is the list of licenses for tools included in this repository, which are us
 
 * :idf:`KConfig <tools/kconfig>` is Copyright (C) 2002 Roman Zippel and others, and is licensed under the GNU General Public License V2.
 
+* :idf:`Kconfiglib <tools/kconfig_new/kconfiglib.py>` is Copyright (C) 2011-2019, Ulf Magnusson, and is licensed under the ISC License.
+
+* :idf:`Menuconfig of Kconfiglib <tools/kconfig_new/menuconfig.py>` is Copyright (C) 2018-2019, Nordic Semiconductor ASA and Ulf Magnusson, and is licensed under the ISC License.
+
 
 ROM Source Code Copyrights
 ==========================

docs/en/api-reference/kconfig.rst

@@ -4,7 +4,12 @@ Project Configuration
 Introduction
 ============
 
-ESP-IDF uses Kconfig_ system to provide a compile-time project configuration mechanism. Kconfig is based around options of several types: integer, string, boolean. Kconfig files specify dependencies between options, default values of the options, the way the options are grouped together, etc.
+ESP-IDF uses kconfiglib_ which is a Python-based extension to the Kconfig_ system which provides a compile-time
+project configuration mechanism. Kconfig is based around options of several types: integer, string, boolean. Kconfig
+files specify dependencies between options, default values of the options, the way the options are grouped together,
+etc.
+
+For the complete list of available features please see Kconfig_ and `kconfiglib extentions`_.
 
 .. _project-configuration-menu:
 
@@ -94,3 +99,5 @@ Because IDF builds by default with :ref:`warn-undefined-variables`, when the Kco
 When generating header files for C & C++, the behaviour is not customised - so ``#ifdef`` can be used to test if a boolean config item is set or not.
 
 .. _Kconfig: https://www.kernel.org/doc/Documentation/kbuild/kconfig-language.txt
+.. _kconfiglib: https://github.com/ulfalizer/Kconfiglib
+.. _kconfiglib extentions: https://pypi.org/project/kconfiglib/#kconfig-extensions

docs/en/get-started-legacy/linux-setup.rst

@@ -12,15 +12,15 @@ To compile with ESP-IDF you need to get the following packages:
 
 - CentOS 7::
 
-    sudo yum install gcc git wget make ncurses-devel flex bison gperf python python2-cryptography
+    sudo yum install gcc git wget make flex bison gperf python python2-cryptography
 
 - Ubuntu and Debian::
 
-    sudo apt-get install gcc git wget make libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing python-pyelftools
+    sudo apt-get install gcc git wget make flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing python-pyelftools
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make ncurses flex bison gperf python2-pyserial python2-cryptography python2-future python2-pyparsing python2-pyelftools
+    sudo pacman -S --needed gcc git make flex bison gperf python2-pyserial python2-cryptography python2-future python2-pyparsing python2-pyelftools
 
 .. note::
 
@@ -86,22 +86,6 @@ Permission issues /dev/ttyUSB0
 
 With some Linux distributions you may get the ``Failed to open port /dev/ttyUSB0`` error message when flashing the ESP32. :ref:`This can be solved by adding the current user to the dialout group<linux-dialout-group-legacy>`.
 
-
-Arch Linux Users
-----------------
-
-To run the precompiled gdb (xtensa-esp32-elf-gdb) in Arch Linux requires ncurses 5, but Arch uses ncurses 6. 
-
-Backwards compatibility libraries are available in AUR_ for native and lib32 configurations:
-
-- https://aur.archlinux.org/packages/ncurses5-compat-libs/
-- https://aur.archlinux.org/packages/lib32-ncurses5-compat-libs/
-
-Before installing these packages you might need to add the author's public key to your keyring as described in the "Comments" section at the links above.
-
-Alternatively, use crosstool-NG to compile a gdb that links against ncurses 6.
-
-
 Next Steps
 ==========
 

docs/en/get-started/linux-setup.rst

@@ -11,15 +11,15 @@ To compile with ESP-IDF you need to get the following packages:
 
 - CentOS 7::
 
-    sudo yum install git wget ncurses-devel flex bison gperf python pyserial python-pyelftools cmake ninja-build ccache
+    sudo yum install git wget flex bison gperf python pyserial python-pyelftools cmake ninja-build ccache
 
 - Ubuntu and Debian::
 
-    sudo apt-get install git wget libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
+    sudo apt-get install git wget flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make ncurses flex bison gperf python2-pip python2-pyserial python2-click python2-cryptography python2-future python2-pyparsing python2-pyelftools cmake ninja ccache
+    sudo pacman -S --needed gcc git make flex bison gperf python2-pip python2-pyserial python2-click python2-cryptography python2-future python2-pyparsing python2-pyelftools cmake ninja ccache
 
 .. note::
     CMake version 3.5 or newer is required for use with ESP-IDF. Older Linux distributions may require updating, enabling of a "backports" repository, or installing of a "cmake3" package rather than "cmake".
@@ -32,20 +32,6 @@ Permission issues /dev/ttyUSB0
 
 With some Linux distributions you may get the ``Failed to open port /dev/ttyUSB0`` error message when flashing the ESP32. :ref:`This can be solved by adding the current user to the dialout group<linux-dialout-group>`.
 
-ncurses 5 dependency
---------------------
-
-To run the precompiled gdb (xtensa-esp32-elf-gdb) in Linux requires ncurses 5, but some newer distributions only provide ncurses 6 by default.
-
-Consult your distribution's documentation to see if ncurses 5 libraries are available. Alternatively, use crosstool-NG to compile a gdb that links against ncurses 6.
-
-For Arch Linux users, ncurses 5 libraries are available in AUR_ for native and lib32 configurations:
-
-- https://aur.archlinux.org/packages/ncurses5-compat-libs/
-- https://aur.archlinux.org/packages/lib32-ncurses5-compat-libs/
-
-Before installing these packages you might need to add the author's public key to your keyring as described in the "Comments" section at the links above.
-
 Next Steps
 ==========
 

docs/zh_CN/get-started-legacy/linux-setup.rst

@@ -12,15 +12,15 @@ Linux 平台工具链的标准设置（传统 GNU Make）
 
 - CentOS 7::
 
-    sudo yum install gcc git wget make ncurses-devel flex bison gperf python python2-cryptography
+    sudo yum install gcc git wget make flex bison gperf python python2-cryptography
 
 - Ubuntu and Debian::
 
-    sudo apt-get install gcc git wget make libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing python-pyelftools
+    sudo apt-get install gcc git wget make flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing python-pyelftools
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make ncurses flex bison gperf python2-pyserial python2-cryptography python2-future python2-pyparsing python2-pyelftools
+    sudo pacman -S --needed gcc git make flex bison gperf python2-pyserial python2-cryptography python2-future python2-pyparsing python2-pyelftools
 
 .. note::
 
@@ -87,21 +87,6 @@ Linux 版的 ESP32 工具链可以从 Espressif 的网站下载：
 某些 Linux 版本可能在烧写 ESP32 时会出现 ``Failed to open port /dev/ttyUSB0`` 错误消息，这可以通过 :ref:`将当前用户添加到 dialout 组 <linux-dialout-group-legacy>` 来解决。
 
 
-Arch Linux 用户
-----------------
-
-在 Arch Linux 中运行预编译 gdb (xtensa-esp32-elf-gdb) 需要 ncurses 5，但 Arch 会使用 ncurses 6。
-
-不过，AUR_ 中有针对原生和 lib32 配置的向下兼容库：
-
-- https://aur.archlinux.org/packages/ncurses5-compat-libs/
-- https://aur.archlinux.org/packages/lib32-ncurses5-compat-libs/
-
-在安装这些软件包之前，您可能需要将作者的公钥添加到您的密钥环中，具体参考上方的 “注释” 部分。
-
-此外，您也可以使用 crosstool-NG 编译一个链接到 ncurses 6 的 gdb。
-
-
 后续步骤
 ==========
 

docs/zh_CN/get-started/linux-setup.rst

@@ -11,15 +11,15 @@ Linux 平台工具链的标准设置
 
 - CentOS 7::
 
-    sudo yum install git wget ncurses-devel flex bison gperf python pyserial python-pyelftools cmake ninja-build ccache
+    sudo yum install git wget flex bison gperf python pyserial python-pyelftools cmake ninja-build ccache
 
 - Ubuntu 和 Debian::
 
-    sudo apt-get install git wget libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
+    sudo apt-get install git wget flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make ncurses flex bison gperf python2-pip python2-pyserial python2-click python2-cryptography python2-future python2-pyparsing python2-pyelftools cmake ninja ccache
+    sudo pacman -S --needed gcc git make flex bison gperf python2-pip python2-pyserial python2-click python2-cryptography python2-future python2-pyparsing python2-pyelftools cmake ninja ccache
 
 .. note::
     使用 ESP-IDF 需要 CMake 3.5 或以上版本。较早版本的 Linux 可能需要升级才能向后移植仓库，或安装 "cmake3" 软件包，而不是安装 "cmake"。
@@ -32,20 +32,6 @@ Linux 平台工具链的标准设置
 
 使用某些 Linux 版本向 ESP32 烧写固件时，可能会出现 ``Failed to open port /dev/ttyUSB0`` 错误消息。此时，可以将当前用户增加至 :ref:` Linux Dialout 组 <linux-dialout-group>`。
 
-ncurses 5 依赖项
---------------------
-
-在 Linux 上运行预编译的 gdb (xtensa-esp32-elf-gdb) 需要 ncurses 5，但一些较新版本默认只提供 ncurses 6。
-
-请查看对应版本信息，确认是否存在可用的 ncurses 5。此外，您也可以使用 crosstool-NG 编译一个链接到 ncurses 6 的 gdb。
-
-Arch Linux 用户可在 AUR_ 中获得 native 和 lib32 配置的 ncurses 5 库：
-
-- https://aur.archlinux.org/packages/ncurses5-compat-libs/
-- https://aur.archlinux.org/packages/lib32-ncurses5-compat-libs/
-
-在安装这些软件包之前，您可能需要将作者的公钥添加到您的密钥环中，具体参考上方的“注释”部分。
-
 后续步骤
 ==========
 

examples/protocols/sockets/udp_multicast/main/Kconfig.projbuild

@@ -24,7 +24,7 @@ menu "Example Configuration"
         bool
     config EXAMPLE_IPV6
         bool
-        select EXAMPLE_CONNECT_IPV6
+        select EXAMPLE_CONNECT_IPV6 if IDF_TARGET_ESP32
 
     config EXAMPLE_MULTICAST_IPV4_ADDR
         string "Multicast IPV4 Address (send & receive)"

make/ldgen.mk

@@ -20,6 +20,8 @@ $(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_librarie
 		--kconfig       $(IDF_PATH)/Kconfig \
 		--env           "COMPONENT_KCONFIGS=$(foreach k, $(COMPONENT_KCONFIGS), $(shell cygpath -m $(k)))" \
 		--env           "COMPONENT_KCONFIGS_PROJBUILD=$(foreach k, $(COMPONENT_KCONFIGS_PROJBUILD), $(shell cygpath -m $(k)))" \
+		--env           "COMPONENT_KCONFIGS_SOURCE_FILE=$(shell cygpath -m $(COMPONENT_KCONFIGS_SOURCE_FILE))" \
+		--env           "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE=$(shell cygpath -m $(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE))" \
 		--env           "IDF_CMAKE=n" \
 		--objdump		$(OBJDUMP)
 endef
@@ -39,6 +41,8 @@ $(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_librarie
 		--kconfig       $(IDF_PATH)/Kconfig \
 		--env           "COMPONENT_KCONFIGS=$(COMPONENT_KCONFIGS)" \
 		--env           "COMPONENT_KCONFIGS_PROJBUILD=$(COMPONENT_KCONFIGS_PROJBUILD)" \
+		--env           "COMPONENT_KCONFIGS_SOURCE_FILE=$(COMPONENT_KCONFIGS_SOURCE_FILE)" \
+		--env           "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE=$(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE)" \
 		--env           "IDF_CMAKE=n" \
 		--objdump		$(OBJDUMP)
 endef

make/project_config.mk

@@ -4,12 +4,17 @@
 COMPONENT_KCONFIGS := $(foreach component,$(COMPONENT_PATHS),$(wildcard $(component)/Kconfig))
 COMPONENT_KCONFIGS_PROJBUILD := $(foreach component,$(COMPONENT_PATHS),$(wildcard $(component)/Kconfig.projbuild))
 COMPONENT_SDKCONFIG_RENAMES := $(foreach component,$(COMPONENT_PATHS),$(wildcard $(component)/sdkconfig.rename))
+COMPONENT_KCONFIGS_SOURCE_FILE:=$(BUILD_DIR_BASE)/kconfigs.in
+COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE:=$(BUILD_DIR_BASE)/kconfigs_projbuild.in
+
 
 ifeq ($(OS),Windows_NT)
 # kconfiglib requires Windows-style paths for kconfig files
 COMPONENT_KCONFIGS := $(shell cygpath -m $(COMPONENT_KCONFIGS))
 COMPONENT_KCONFIGS_PROJBUILD := $(shell cygpath -m $(COMPONENT_KCONFIGS_PROJBUILD))
 COMPONENT_SDKCONFIG_RENAMES := $(shell cygpath -m $(COMPONENT_SDKCONFIG_RENAMES))
+COMPONENT_KCONFIGS_SOURCE_FILE := $(shell cygpath -m $(COMPONENT_KCONFIGS_SOURCE_FILE))
+COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE := $(shell cygpath -m $(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE))
 endif
 
 #For doing make menuconfig etc
@@ -30,8 +35,13 @@ $(KCONFIG_TOOL_DIR)/mconf-idf: $(KCONFIG_TOOL_DIR)/conf-idf
 
 # reset MAKEFLAGS as the menuconfig makefile uses implicit compile rules
 $(KCONFIG_TOOL_DIR)/mconf-idf $(KCONFIG_TOOL_DIR)/conf-idf: $(wildcard $(KCONFIG_TOOL_DIR)/*.c)
+ifeq ($(OS),Windows_NT)
+	# mconf-idf is used only in MSYS
 	MAKEFLAGS="" CC=$(HOSTCC) LD=$(HOSTLD) \
 	$(MAKE) -C $(KCONFIG_TOOL_DIR)
+else
+	@echo "mconf-idf is not required on this platform"
+endif
 
 ifeq ("$(wildcard $(SDKCONFIG))","")
 # if no configuration file is present we need a rule for it
@@ -46,6 +56,11 @@ $(SDKCONFIG): defconfig
 endif
 endif
 
+ifeq ("$(PYTHON)","")
+# fallback value when menuconfig is called without a build directory and sdkconfig file
+PYTHON=python
+endif
+
 ifneq ("$(wildcard $(SDKCONFIG_DEFAULTS))","")
 ifeq ($(OS),Windows_NT)
 DEFAULTS_ARG:=--defaults $(shell cygpath -m $(SDKCONFIG_DEFAULTS))
@@ -65,6 +80,8 @@ define RunConfGen
 		--sdkconfig-rename $(SDKCONFIG_RENAME) \
 		--env "COMPONENT_KCONFIGS=$(strip $(COMPONENT_KCONFIGS))" \
 		--env "COMPONENT_KCONFIGS_PROJBUILD=$(strip $(COMPONENT_KCONFIGS_PROJBUILD))" \
+		--env "COMPONENT_KCONFIGS_SOURCE_FILE=$(COMPONENT_KCONFIGS_SOURCE_FILE)" \
+		--env "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE=$(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE)" \
 		--env "COMPONENT_SDKCONFIG_RENAMES=$(strip $(COMPONENT_SDKCONFIG_RENAMES))" \
 		--env "IDF_CMAKE=n" \
 		$(DEFAULTS_ARG) \
@@ -73,17 +90,21 @@ define RunConfGen
 		--output header $(BUILD_DIR_BASE)/include/sdkconfig.h
 endef
 
-# macro for the commands to run kconfig tools conf-idf or mconf-idf.
+ifeq ($(OS),Windows_NT)
-# $1 is the name (& args) of the conf tool to run
+MENUCONFIG_CMD := $(KCONFIG_TOOL_DIR)/mconf-idf
-# Note: Currently only mconf-idf is used for compatibility with the CMake build system. The header file used is also
+else
-# the same.
+MENUCONFIG_CMD := MENUCONFIG_STYLE=aquatic $(PYTHON) $(IDF_PATH)/tools/kconfig_new/menuconfig.py
-define RunConf
+endif
+
+# macro for running menuconfig
+define RunMenuConf
 	mkdir -p $(BUILD_DIR_BASE)/include/config
 	cd $(BUILD_DIR_BASE); KCONFIG_AUTOHEADER=$(abspath $(BUILD_DIR_BASE)/include/sdkconfig.h) \
-	COMPONENT_KCONFIGS="$(COMPONENT_KCONFIGS)" KCONFIG_CONFIG=$(SDKCONFIG) \
+	KCONFIG_CONFIG=$(SDKCONFIG) \
-	COMPONENT_KCONFIGS_PROJBUILD="$(COMPONENT_KCONFIGS_PROJBUILD)" \
+	COMPONENT_KCONFIGS_SOURCE_FILE="$(COMPONENT_KCONFIGS_SOURCE_FILE)" \
+	COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE="$(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE)" \
 	IDF_CMAKE=n \
-	$(KCONFIG_TOOL_DIR)/$1 $(IDF_PATH)/Kconfig
+	$(MENUCONFIG_CMD) $(IDF_PATH)/Kconfig
 endef
 
 ifndef MAKE_RESTARTS
@@ -108,9 +129,9 @@ ifdef BATCH_BUILD
 	@exit 1
 else
 	$(call RunConfGen)
-	# RunConfGen before mconf-idf ensures that deprecated options won't be ignored (they've got renamed)
+	# RunConfGen before menuconfig ensures that deprecated options won't be ignored (they've got renamed)
-	$(call RunConf,mconf-idf)
+	$(call RunMenuConf)
-	# RunConfGen after mconf-idf ensures that deprecated options are appended to $(SDKCONFIG) for backward compatibility
+	# RunConfGen after menuconfig ensures that deprecated options are appended to $(SDKCONFIG) for backward compatibility
 	$(call RunConfGen)
 endif
 
@@ -136,4 +157,5 @@ endif
 config-clean:
 	$(summary) RM CONFIG
 	MAKEFLAGS="" $(MAKE) -C $(KCONFIG_TOOL_DIR) clean
-	rm -rf $(BUILD_DIR_BASE)/include/config $(BUILD_DIR_BASE)/include/sdkconfig.h
+	rm -rf $(BUILD_DIR_BASE)/include/config $(BUILD_DIR_BASE)/include/sdkconfig.h \
+		$(COMPONENT_KCONFIGS_SOURCE_FILE) $(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE)

requirements.txt

@@ -11,3 +11,8 @@ future>=0.15.2
 cryptography>=2.1.4
 pyparsing>=2.0.3,<2.4.0
 pyelftools>=0.22
+# windows-curses are required in Windows command line but cannot be installed in MSYS2. A requirement like
+# "windows-curses; sys_platform == 'win32'" would want to install the package on both of them. There is no environment
+# marker for detecting MSYS2. So instead, a dummy custom package is used with "windows-curses" dependency for Windows
+# command line.
+file://${IDF_PATH}/tools/kconfig_new/esp-windows-curses; sys_platform == 'win32'

tools/check_kconfigs.py

@@ -95,7 +95,7 @@ class SourceChecker(BaseChecker):
                                  line.replace('source', 'source '))
             path = m.group(2)
             filename = os.path.basename(path)
-            if path in ['$COMPONENT_KCONFIGS_PROJBUILD', '$COMPONENT_KCONFIGS']:
+            if path in ['$COMPONENT_KCONFIGS_SOURCE_FILE', '$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE']:
                 pass
             elif not filename.startswith('Kconfig.'):
                 raise InputError(self.path_in_idf, line_number, "only filenames starting with Kconfig.* can be sourced",

tools/check_python_dependencies.py

@@ -55,6 +55,10 @@ if __name__ == "__main__":
     with open(args.requirements) as f:
         for line in f:
             line = line.strip()
+            # pkg_resources.require() cannot handle the full requirements file syntax so we need to make
+            # adjustments for options which we use.
+            if line.startswith('file://'):
+                line = os.path.basename(line)
             try:
                 pkg_resources.require(line)
             except Exception:

tools/cmake/kconfig.cmake

@@ -1,14 +1,16 @@
 include(ExternalProject)
 
 function(__kconfig_init)
+    if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "FreeBSD")
+        set(MAKE_COMMMAND "gmake")
+    else()
+        set(MAKE_COMMMAND "make")
+    endif()
+
     idf_build_get_property(idf_path IDF_PATH)
-    if(CMAKE_HOST_WIN32)
+    if(CMAKE_HOST_WIN32 AND DEFINED ENV{MSYSTEM})
         # Prefer a prebuilt mconf-idf on Windows
-        if(DEFINED ENV{MSYSTEM})
+        find_program(WINPTY winpty)
-            find_program(WINPTY winpty)
-        else()
-            unset(WINPTY CACHE)  # in case previous CMake run was in a tty and this one is not
-        endif()
         unset(MCONF CACHE)  # needed when MSYS and CMD is intermixed (cache could contain an incompatible path)
         find_program(MCONF mconf-idf)
 
@@ -26,9 +28,38 @@ function(__kconfig_init)
             find_program(NATIVE_GCC gcc)
             if(NOT NATIVE_GCC)
                 message(FATAL_ERROR
-                    "Windows requires a prebuilt mconf-idf for your platform "
+                    "Windows requires an MSYS2 version of gcc on the PATH to build mconf-idf. "
-                    "on the PATH, or an MSYS2 version of gcc on the PATH to build mconf-idf. "
                     "Consult the setup docs for ESP-IDF on Windows.")
+            else()
+                # Use the existing Makefile to build mconf (out of tree) when needed
+                #
+                set(MCONF ${CMAKE_BINARY_DIR}/kconfig_bin/mconf-idf)
+                set(src_path ${idf_path}/tools/kconfig)
+
+                # note: we preemptively remove any build files from the src dir
+                # as we're building out of tree, but don't want build system to
+                # #include any from there that were previously build with/for make
+                externalproject_add(mconf-idf
+                    SOURCE_DIR ${src_path}
+                    CONFIGURE_COMMAND ""
+                    BINARY_DIR "${CMAKE_BINARY_DIR}/kconfig_bin"
+                    BUILD_COMMAND rm -f ${src_path}/zconf.lex.c ${src_path}/zconf.hash.c
+                    COMMAND ${MAKE_COMMMAND} -f ${src_path}/Makefile mconf-idf
+                    BUILD_BYPRODUCTS ${MCONF}
+                    INSTALL_COMMAND ""
+                    EXCLUDE_FROM_ALL 1
+                    )
+
+                file(GLOB mconf_srcfiles ${src_path}/*.c)
+                list(REMOVE_ITEM mconf_srcfiles "${src_path}/zconf.lex.c" "${src_path}/zconf.hash.c")
+                externalproject_add_stepdependencies(mconf-idf build
+                    ${mconf_srcfiles}
+                    ${src_path}/Makefile
+                    ${CMAKE_CURRENT_LIST_FILE})
+                unset(mconf_srcfiles)
+                unset(src_path)
+
+                set(menuconfig_depends DEPENDS mconf-idf)
             endif()
         else()
             execute_process(COMMAND "${MCONF}" -v
@@ -53,47 +84,9 @@ function(__kconfig_init)
                 set(MCONF "\"${WINPTY}\" \"${MCONF}\"")
             endif()
         endif()
+        idf_build_set_property(__MCONF ${MCONF})
+        idf_build_set_property(__MENUCONFIG_DEPENDS "${menuconfig_depends}")
     endif()
-    if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "FreeBSD")
-        set(MAKE_COMMMAND "gmake")
-    else()
-        set(MAKE_COMMMAND "make")
-    endif()
-
-    if(NOT MCONF)
-        # Use the existing Makefile to build mconf (out of tree) when needed
-        #
-        set(MCONF ${CMAKE_BINARY_DIR}/kconfig_bin/mconf-idf)
-        set(src_path ${idf_path}/tools/kconfig)
-
-        # note: we preemptively remove any build files from the src dir
-        # as we're building out of tree, but don't want build system to
-        # #include any from there that were previously build with/for make
-        externalproject_add(mconf-idf
-            SOURCE_DIR ${src_path}
-            CONFIGURE_COMMAND ""
-            BINARY_DIR "${CMAKE_BINARY_DIR}/kconfig_bin"
-            BUILD_COMMAND rm -f ${src_path}/zconf.lex.c ${src_path}/zconf.hash.c
-            COMMAND ${MAKE_COMMMAND} -f ${src_path}/Makefile mconf-idf
-            BUILD_BYPRODUCTS ${MCONF}
-            INSTALL_COMMAND ""
-            EXCLUDE_FROM_ALL 1
-            )
-
-        file(GLOB mconf_srcfiles ${src_path}/*.c)
-        list(REMOVE_ITEM mconf_srcfiles "${src_path}/zconf.lex.c" "${src_path}/zconf.hash.c")
-        externalproject_add_stepdependencies(mconf-idf build
-            ${mconf_srcfiles}
-            ${src_path}/Makefile
-            ${CMAKE_CURRENT_LIST_FILE})
-        unset(mconf_srcfiles)
-        unset(src_path)
-
-        set(menuconfig_depends DEPENDS mconf-idf)
-    endif()
-
-    idf_build_set_property(__MCONF ${MCONF})
-    idf_build_set_property(__MENUCONFIG_DEPENDS "${menuconfig_depends}")
 
     idf_build_get_property(idf_path IDF_PATH)
     idf_build_set_property(__ROOT_KCONFIG ${idf_path}/Kconfig)
@@ -152,6 +145,11 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
     string(REPLACE ";" " " kconfig_projbuilds "${kconfig_projbuilds}")
     string(REPLACE ";" " " sdkconfig_renames "${sdkconfig_renames}")
 
+    # These are the paths for files which will contain the generated "source" lines for COMPONENT_KCONFIGS and
+    # COMPONENT_KCONFIGS_PROJBUILD
+    set(kconfigs_projbuild_path "${CMAKE_CURRENT_BINARY_DIR}/kconfigs_projbuild.in")
+    set(kconfigs_path "${CMAKE_CURRENT_BINARY_DIR}/kconfigs.in")
+
     # Place config-related environment arguments into config.env file
     # to work around command line length limits for execute_process
     # on Windows & CMake < 3.11
@@ -237,23 +235,28 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
     idf_build_set_property(SDKCONFIG_JSON_MENUS ${sdkconfig_json_menus})
     idf_build_set_property(CONFIG_DIR ${config_dir})
 
-    idf_build_get_property(menuconfig_depends __MENUCONFIG_DEPENDS)
+    if(CMAKE_HOST_WIN32 AND DEFINED ENV{MSYSTEM})
+        idf_build_get_property(menuconfig_depends __MENUCONFIG_DEPENDS)
+        idf_build_get_property(mconf __MCONF)
 
-    idf_build_get_property(mconf __MCONF)
+        set(MENUCONFIG_CMD ${mconf})
+    else()
+        set(MENUCONFIG_CMD "MENUCONFIG_STYLE=aquatic" ${python} ${idf_path}/tools/kconfig_new/menuconfig.py)
+    endif()
 
-    # Generate the menuconfig target (uses C-based mconf-idf tool, either prebuilt or via mconf-idf target above)
+    # Generate the menuconfig target
     add_custom_target(menuconfig
         ${menuconfig_depends}
         # create any missing config file, with defaults if necessary
         COMMAND ${confgen_basecommand} --env "IDF_TARGET=${idf_target}" --output config ${sdkconfig}
         COMMAND ${CMAKE_COMMAND} -E env
-        "COMPONENT_KCONFIGS=${kconfigs}"
+        "COMPONENT_KCONFIGS_SOURCE_FILE=${kconfigs_path}"
-        "COMPONENT_KCONFIGS_PROJBUILD=${kconfig_projbuilds}"
+        "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE=${kconfigs_projbuild_path}"
         "IDF_CMAKE=y"
         "IDF_TARGET=${IDF_TARGET}"
         "KCONFIG_CONFIG=${sdkconfig}"
         "IDF_TARGET=${idf_target}"
-        ${mconf} ${root_kconfig}
+        ${MENUCONFIG_CMD} ${root_kconfig}
         # VERBATIM cannot be used here because it cannot handle ${mconf}="winpty mconf-idf" and the escaping must be
         # done manually
         USES_TERMINAL

tools/kconfig/confdata.c

@@ -489,9 +489,6 @@ kconfig_print_symbol(FILE *fp, struct symbol *sym, const char *value, void *arg)
 	switch (sym->type) {
 	case S_BOOLEAN:
 	case S_TRISTATE:
-		if (*value == 'n') {
-			value = "";
-		}
 		break;
 	default:
 		break;

tools/kconfig/lxdialog/check-lxdialog.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 # Check ncurses compatibility
 
+if [ "$OSTYPE" != "msys" ]; then
+  echo "The old menuconfig is expected to be built only in MSYS2. Please report this issue if you encounter it." >&2
+  exit 1
+fi
+
 # What library to link
 ldflags()
 {
@@ -12,8 +17,6 @@ ldflags()
 		# libintl
 		echo -n "-L/usr/local/lib -lintl "
 	fi
-	pkg-config --libs ncursesw 2>/dev/null && exit
-	pkg-config --libs ncurses 2>/dev/null && exit
 	for ext in so a dll.a dylib ; do
 		for lib in ncursesw ncurses curses ; do
 			$cc -print-file-name=lib${lib}.${ext} $extra_libs | grep -q /
@@ -29,11 +32,10 @@ ldflags()
 # Where is ncurses.h?
 ccflags()
 {
-	if pkg-config --cflags ncursesw 2>/dev/null; then
+    # pkg-config doesn't exist in older MSYS (ESP-IDF v20190611). In newer environment, it will find ncurses bundled with MINGW
-		echo '-DCURSES_LOC="<ncurses.h>" -DNCURSES_WIDECHAR=1'
+    # Pythons and the compilation will fail.
-	elif pkg-config --cflags ncurses 2>/dev/null; then
+    # NOTE: Only MSYS is using tools/kconfig.
-		echo '-DCURSES_LOC="<ncurses.h>"'
+	if [ -f /usr/include/ncursesw/curses.h ]; then
-	elif [ -f /usr/include/ncursesw/curses.h ]; then
 		echo '-I/usr/include/ncursesw -DCURSES_LOC="<curses.h>"'
 		echo ' -DNCURSES_WIDECHAR=1'
 	elif [ -f /usr/include/ncurses/ncurses.h ]; then

tools/kconfig_new/confgen.py

@@ -28,6 +28,7 @@ import os.path
 import re
 import sys
 import tempfile
+from future.utils import iteritems
 
 import gen_kconfig_doc
 
@@ -150,7 +151,8 @@ class DeprecatedOptions(object):
                         tmp_list.append(c_string.replace(self.config_prefix + item.name,
                                                          self.config_prefix + self.rev_r_dic[item.name]))
 
-        config.walk_menu(append_config_node_process)
+        for n in config.node_iter():
+            append_config_node_process(n)
 
         if len(tmp_list) > 0:
             with open(path_output, 'a') as f_o:
@@ -173,6 +175,42 @@ class DeprecatedOptions(object):
                         f_o.write('#define {}{} {}{}\n'.format(self.config_prefix, dep_opt, self.config_prefix, new_opt))
 
 
+def prepare_source_files():
+    """
+    Prepares source files which are sourced from the main Kconfig because upstream kconfiglib doesn't support sourcing
+    a file list.
+    """
+
+    def _dequote(var):
+        return var[1:-1] if len(var) > 0 and (var[0], var[-1]) == ('"',) * 2 else var
+
+    def _write_source_file(config_var, config_file):
+        with open(config_file, "w") as f:
+            f.write('\n'.join(['source "{}"'.format(path) for path in _dequote(config_var).split()]))
+
+    try:
+        _write_source_file(os.environ['COMPONENT_KCONFIGS'], os.environ['COMPONENT_KCONFIGS_SOURCE_FILE'])
+        _write_source_file(os.environ['COMPONENT_KCONFIGS_PROJBUILD'], os.environ['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE'])
+    except KeyError as e:
+        print('Error:', e, 'is not defined!')
+        raise
+
+
+def dict_enc_for_env(dic, encoding=sys.getfilesystemencoding() or 'utf-8'):
+    """
+    This function can be deleted after dropping support for Python 2.
+    There is no rule for it that environment variables cannot be Unicode but usually people try to avoid it.
+    The upstream kconfiglib cannot detect strings properly if the environment variables are "unicode". This is problem
+    only in Python 2.
+    """
+    if sys.version_info[0] >= 3:
+        return dic
+    ret = dict()
+    for (key, value) in iteritems(dic):
+        ret[key.encode(encoding)] = value.encode(encoding)
+    return ret
+
+
 def main():
     parser = argparse.ArgumentParser(description='confgen.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0]))
 
@@ -226,17 +264,32 @@ def main():
 
     if args.env_file is not None:
         env = json.load(args.env_file)
-        os.environ.update(env)
+        os.environ.update(dict_enc_for_env(env))
 
+    prepare_source_files()
     config = kconfiglib.Kconfig(args.kconfig)
-    config.disable_redun_warnings()
+    config.warn_assign_redun = False
-    config.disable_override_warnings()
+    config.warn_assign_override = False
+
+    sdkconfig_renames = [args.sdkconfig_rename] if args.sdkconfig_rename else []
+    sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()
+    deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames)
 
     sdkconfig_renames = [args.sdkconfig_rename] if args.sdkconfig_rename else []
     sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()
     deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames)
 
     if len(args.defaults) > 0:
+        def _replace_empty_assignments(path_in, path_out):
+            with open(path_in, 'r') as f_in, open(path_out, 'w') as f_out:
+                for line_num, line in enumerate(f_in, start=1):
+                    line = line.strip()
+                    if line.endswith('='):
+                        line += 'n'
+                        print('{}:{} line was updated to {}'.format(path_out, line_num, line))
+                    f_out.write(line)
+                    f_out.write('\n')
+
         # always load defaults first, so any items which are not defined in that config
         # will have the default defined in the defaults file
         for name in args.defaults:
@@ -245,12 +298,16 @@ def main():
                 raise RuntimeError("Defaults file not found: %s" % name)
             try:
                 with tempfile.NamedTemporaryFile(prefix="confgen_tmp", delete=False) as f:
-                    temp_file = f.name
+                    temp_file1 = f.name
-                deprecated_options.replace(sdkconfig_in=name, sdkconfig_out=temp_file)
+                with tempfile.NamedTemporaryFile(prefix="confgen_tmp", delete=False) as f:
-                config.load_config(temp_file, replace=False)
+                    temp_file2 = f.name
+                deprecated_options.replace(sdkconfig_in=name, sdkconfig_out=temp_file1)
+                _replace_empty_assignments(temp_file1, temp_file2)
+                config.load_config(temp_file2, replace=False)
             finally:
                 try:
-                    os.remove(temp_file)
+                    os.remove(temp_file1)
+                    os.remove(temp_file2)
                 except OSError:
                     pass
 
@@ -326,7 +383,8 @@ def write_makefile(deprecated_options, config, filename):
                     # the same string but with the deprecated name
                     tmp_dep_lines.append(get_makefile_config_string(dep_opt, val, item.orig_type))
 
-        config.walk_menu(write_makefile_node, True)
+        for n in config.node_iter(True):
+            write_makefile_node(n)
 
         if len(tmp_dep_lines) > 0:
             f.write('\n# List of deprecated options\n')
@@ -376,7 +434,8 @@ def write_cmake(deprecated_options, config, filename):
                     tmp_dep_list.append("set({}{} \"{}\")\n".format(prefix, dep_opt, val))
                     configs_list.append(prefix + dep_opt)
 
-        config.walk_menu(write_node)
+        for n in config.node_iter():
+            write_node(n)
         write("set(CONFIGS_LIST {})".format(";".join(configs_list)))
 
         if len(tmp_dep_list) > 0:
@@ -401,7 +460,8 @@ def get_json_values(config):
             elif sym.type == kconfiglib.INT:
                 val = int(val)
             config_dict[sym.name] = val
-    config.walk_menu(write_node)
+    for n in config.node_iter(False):
+        write_node(n)
     return config_dict
 
 
@@ -454,7 +514,8 @@ def write_json_menus(deprecated_options, config, filename):
             depends = kconfiglib.expr_str(node.dep)
 
         try:
-            is_menuconfig = node.is_menuconfig
+            # node.is_menuconfig is True in newer kconfiglibs for menus and choices as well
+            is_menuconfig = node.is_menuconfig and isinstance(node.item, kconfiglib.Symbol)
         except AttributeError:
             is_menuconfig = False
 
@@ -521,7 +582,8 @@ def write_json_menus(deprecated_options, config, filename):
             json_parent.append(new_json)
             node_lookup[node] = new_json
 
-    config.walk_menu(write_node)
+    for n in config.node_iter():
+        write_node(n)
     with open(filename, "w") as f:
         f.write(json.dumps(result, sort_keys=True, indent=4))
 

tools/kconfig_new/config.env.in

@@ -4,5 +4,7 @@
     "COMPONENT_SDKCONFIG_RENAMES": "${sdkconfig_renames}",
     "IDF_CMAKE": "y",
     "IDF_TARGET": "${idf_target}",
-    "IDF_PATH": "${idf_path}"
+    "IDF_PATH": "${idf_path}",
+    "COMPONENT_KCONFIGS_SOURCE_FILE": "${kconfigs_path}",
+    "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE": "${kconfigs_projbuild_path}"
 }

tools/kconfig_new/confserver.py

@@ -69,12 +69,13 @@ def main():
 
     if args.env_file is not None:
         env = json.load(args.env_file)
-        os.environ.update(env)
+        os.environ.update(confgen.dict_enc_for_env(env))
 
     run_server(args.kconfig, args.config, args.sdkconfig_rename)
 
 
 def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCOL_VERSION):
+    confgen.prepare_source_files()
     config = kconfiglib.Kconfig(kconfig)
     sdkconfig_renames = [sdkconfig_rename] if sdkconfig_rename else []
     sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()
@@ -245,15 +246,41 @@ def diff(before, after):
 def get_ranges(config):
     ranges_dict = {}
 
+    def is_base_n(i, n):
+        try:
+            int(i, n)
+            return True
+        except ValueError:
+            return False
+
+    def get_active_range(sym):
+        """
+        Returns a tuple of (low, high) integer values if a range
+        limit is active for this symbol, or (None, None) if no range
+        limit exists.
+        """
+        base = kconfiglib._TYPE_TO_BASE[sym.orig_type] if sym.orig_type in kconfiglib._TYPE_TO_BASE else 0
+
+        try:
+            for low_expr, high_expr, cond in sym.ranges:
+                if kconfiglib.expr_value(cond):
+                    low = int(low_expr.str_value, base) if is_base_n(low_expr.str_value, base) else 0
+                    high = int(high_expr.str_value, base) if is_base_n(high_expr.str_value, base) else 0
+                    return (low, high)
+        except ValueError:
+            pass
+        return (None, None)
+
     def handle_node(node):
         sym = node.item
         if not isinstance(sym, kconfiglib.Symbol):
             return
-        active_range = sym.active_range
+        active_range = get_active_range(sym)
         if active_range[0] is not None:
             ranges_dict[sym.name] = active_range
 
-    config.walk_menu(handle_node)
+    for n in config.node_iter():
+        handle_node(n)
     return ranges_dict
 
 
@@ -274,7 +301,8 @@ def get_visible(config):
             result[node] = visible
         except AttributeError:
             menus.append(node)
-    config.walk_menu(handle_node)
+    for n in config.node_iter():
+        handle_node(n)
 
     # now, figure out visibility for each menu. A menu is visible if any of its children are visible
     for m in reversed(menus):  # reverse to start at leaf nodes

tools/kconfig_new/esp-windows-curses/setup.py

@@ -0,0 +1,30 @@
+# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from setuptools import setup
+
+setup(name='esp-windows-curses',
+      version='0.1',
+      description='Wrapper for the windows-curses package',
+      url='https://www.espressif.com',
+      author='Espressif Systems',
+      license='Apache License 2.0',
+      author_email='donotreply@espressif.com',
+      zip_safe=False,
+      # This wrapper exists only because of the following install_requires statement which ensures that the package
+      # dependency is not added for MSYS2 where it cannot be installed. There is no PEP 508 environment marker to
+      # detect MSYS2.
+      install_requires=('' if 'MSYSTEM' in os.environ else 'windows-curses; sys_platform == "win32"',)
+      )

tools/kconfig_new/gen_kconfig_doc.py

@@ -47,7 +47,8 @@ def write_docs(config, filename):
     of any items. ie the --config option can be ignored.
     (However at time of writing it still needs to be set to something...) """
     with open(filename, "w") as f:
-        config.walk_menu(lambda node: write_menu_item(f, node))
+        for node in config.node_iter():
+            write_menu_item(f, node)
 
 
 def node_is_menu(node):
@@ -150,6 +151,7 @@ def write_menu_item(f, node):
             # each line are stripped by kconfiglib. We need to re-indent the text
             # to produce valid ReST.
             f.write(format_rest_text(node.help, INDENT))
+            f.write('\n')
     except AttributeError:
         pass  # No help
 

tools/kconfig_new/test/test_confserver.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 from __future__ import print_function
-import os
-import json
 import argparse
+import json
+import os
+import re
 import tempfile
 
 import pexpect
@@ -49,7 +50,22 @@ def main():
             with open("sdkconfig") as orig:
                 temp_sdkconfig.write(orig.read())
 
-        cmdline = "../confserver.py --kconfig Kconfig --config %s" % temp_sdkconfig_path
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            temp_kconfigs_source_file = os.path.join(tempfile.gettempdir(), f.name)
+
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            temp_kconfig_projbuilds_source_file = os.path.join(tempfile.gettempdir(), f.name)
+
+        cmdline = '''../confserver.py --env "COMPONENT_KCONFIGS_SOURCE_FILE=%s" \
+                                      --env "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE=%s" \
+                                      --env "COMPONENT_KCONFIGS=" \
+                                      --env "COMPONENT_KCONFIGS_PROJBUILD=" \
+                                      --kconfig Kconfig \
+                                      --config %s \
+                  ''' % (temp_kconfigs_source_file, temp_kconfig_projbuilds_source_file, temp_sdkconfig_path)
+
+        cmdline = re.sub(r' +', ' ', cmdline)
+
         print("Running: %s" % cmdline)
         p = pexpect.spawn(cmdline, timeout=30, logfile=args.logfile, echo=False, use_poll=True, maxread=1)
 
@@ -69,6 +85,8 @@ def main():
     finally:
         try:
             os.remove(temp_sdkconfig_path)
+            os.remove(temp_kconfigs_source_file)
+            os.remove(temp_kconfig_projbuilds_source_file)
         except OSError:
             pass
 

tools/ldgen/ldgen.py

@@ -30,6 +30,14 @@ from ldgen_common import LdGenFailure
 from pyparsing import ParseException, ParseFatalException
 from io import StringIO
 
+try:
+    import confgen
+except Exception:
+    parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    kconfig_new_dir = os.path.abspath(parent_dir_name + "/kconfig_new")
+    sys.path.insert(0, kconfig_new_dir)
+    import confgen
+
 
 def _update_environment(args):
     env = [(name, value) for (name,value) in (e.split("=",1) for e in args.env)]
@@ -39,7 +47,7 @@ def _update_environment(args):
 
     if args.env_file is not None:
         env = json.load(args.env_file)
-        os.environ.update(env)
+        os.environ.update(confgen.dict_enc_for_env(env))
 
 
 def main():

tools/ldgen/sdkconfig.py

@@ -26,6 +26,8 @@ except Exception:
     sys.path.insert(0, kconfig_new_dir)
     import kconfiglib
 
+import confgen
+
 
 class SDKConfig:
     """
@@ -47,6 +49,7 @@ class SDKConfig:
     OPERATOR = oneOf(["=", "!=", ">", "<", "<=", ">="])
 
     def __init__(self, kconfig_file, sdkconfig_file):
+        confgen.prepare_source_files()
         self.config = kconfiglib.Kconfig(kconfig_file)
         self.config.load_config(sdkconfig_file)
 

tools/ldgen/test/test_fragments.py

@@ -14,8 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import os
 import sys
 import unittest
+import tempfile
 
 from io import StringIO
 from pyparsing import Word, ParseException, ParseFatalException, alphanums
@@ -57,8 +59,25 @@ FRAGMENT_TYPES["test"] = SampleFragment
 class FragmentTest(unittest.TestCase):
 
     def setUp(self):
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            self.kconfigs_source_file = os.path.join(tempfile.gettempdir(), f.name)
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            self.kconfig_projbuilds_source_file = os.path.join(tempfile.gettempdir(), f.name)
+
+        os.environ['COMPONENT_KCONFIGS_SOURCE_FILE'] = self.kconfigs_source_file
+        os.environ['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE'] = self.kconfig_projbuilds_source_file
+        os.environ['COMPONENT_KCONFIGS'] = ''
+        os.environ['COMPONENT_KCONFIGS_PROJBUILD'] = ''
+
         self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
 
+    def tearDown(self):
+        try:
+            os.remove(self.kconfigs_source_file)
+            os.remove(self.kconfig_projbuilds_source_file)
+        except Exception:
+            pass
+
     @staticmethod
     def create_fragment_file(contents, name="test_fragment.lf"):
         f = StringIO(contents)

tools/ldgen/test/test_generation.py

@@ -15,8 +15,10 @@
 # limitations under the License.
 #
 
-import unittest
+import os
 import sys
+import tempfile
+import unittest
 
 try:
     from generation import PlacementRule
@@ -42,6 +44,18 @@ class GenerationModelTest(unittest.TestCase):
         self.sections_info = None
         self.script_model = None
 
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            self.kconfigs_source_file = os.path.join(tempfile.gettempdir(), f.name)
+            self.addCleanup(os.remove, self.kconfigs_source_file)
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            self.kconfig_projbuilds_source_file = os.path.join(tempfile.gettempdir(), f.name)
+            self.addCleanup(os.remove, self.kconfig_projbuilds_source_file)
+
+        os.environ['COMPONENT_KCONFIGS_SOURCE_FILE'] = self.kconfigs_source_file
+        os.environ['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE'] = self.kconfig_projbuilds_source_file
+        os.environ['COMPONENT_KCONFIGS'] = ''
+        os.environ['COMPONENT_KCONFIGS_PROJBUILD'] = ''
+
         self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
 
         with open("data/sample.lf") as fragment_file_obj:

tools/tools.json

@@ -292,50 +292,6 @@
         }
       ]
     },
-    {
-      "description": "menuconfig tool",
-      "export_paths": [
-        [
-          ""
-        ]
-      ],
-      "export_vars": {},
-      "info_url": "https://github.com/espressif/kconfig-frontends",
-      "install": "never",
-      "license": "GPL-2.0-only",
-      "name": "mconf",
-      "platform_overrides": [
-        {
-          "install": "always",
-          "platforms": [
-            "win32",
-            "win64"
-          ]
-        }
-      ],
-      "strip_container_dirs": 1,
-      "version_cmd": [
-        "mconf-idf",
-        "-v"
-      ],
-      "version_regex": "mconf-idf version mconf-([a-z0-9.-]+)-win32",
-      "versions": [
-        {
-          "name": "v4.6.0.0-idf-20190628",
-          "status": "recommended",
-          "win32": {
-            "sha256": "1b8f17f48740ab669c13bd89136e8cc92efe0cd29872f0d6c44148902a2dc40c",
-            "size": 826114,
-            "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190628/mconf-v4.6.0.0-idf-20190628-win32.zip"
-          },
-          "win64": {
-            "sha256": "1b8f17f48740ab669c13bd89136e8cc92efe0cd29872f0d6c44148902a2dc40c",
-            "size": 826114,
-            "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190628/mconf-v4.6.0.0-idf-20190628-win32.zip"
-          }
-        }
-      ]
-    },
     {
       "description": "Ninja build system",
       "export_paths": [