From b76a10dd4f7de9e121609983aa0374a7a603d79b Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Thu, 30 Jan 2020 17:18:20 +0100 Subject: [PATCH 1/4] idf.py: guess IDF_TARGET from sdkconfig if CMakeCache doesn't exist This solves the issue that target is changed to the default one after idf.py fullclean. Also allow setting the default target using sdkconfig.defaults, e.g. CONFIG_IDF_TARGET="esp32s2" Closes IDF-1040 --- tools/ci/test_build_system_cmake.sh | 28 +++++++++++++++++++++++++ tools/idf_py_actions/tools.py | 32 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/tools/ci/test_build_system_cmake.sh b/tools/ci/test_build_system_cmake.sh index f22446d75..a8d3adf94 100755 --- a/tools/ci/test_build_system_cmake.sh +++ b/tools/ci/test_build_system_cmake.sh @@ -325,6 +325,34 @@ function run_tests() grep "CONFIG_IDF_TARGET=\"${other_target}\"" sdkconfig || failure "Project not configured correctly using idf.py set-target" grep "IDF_TARGET:STRING=${other_target}" build/CMakeCache.txt || failure "IDF_TARGET not set in CMakeCache.txt using idf.py set-target" + print_status "Can guess target from sdkconfig, if CMakeCache does not exist" + idf.py fullclean || failure "Failed to clean the build directory" + idf.py reconfigure || failure "Failed to reconfigure after fullclean" + grep "CONFIG_IDF_TARGET=\"${other_target}\"" sdkconfig || failure "Didn't find the expected CONFIG_IDF_TARGET value" + grep "IDF_TARGET:STRING=${other_target}" build/CMakeCache.txt || failure "IDF_TARGET not set in CMakeCache.txt after fullclean and reconfigure" + + print_status "Can set the default target using sdkconfig.defaults" + clean_build_dir + rm sdkconfig + echo "CONFIG_IDF_TARGET=\"${other_target}\"" > sdkconfig.defaults + idf.py reconfigure || failure "Failed to reconfigure with default target set in sdkconfig.defaults" + grep "CONFIG_IDF_TARGET=\"${other_target}\"" sdkconfig || failure "Didn't find the expected CONFIG_IDF_TARGET value" + grep "CONFIG_IDF_TARGET_${other_target^^}=y" sdkconfig || failure "Didn't find CONFIG_IDF_TARGET_${other_target^^} value" + grep "IDF_TARGET:STRING=${other_target}" build/CMakeCache.txt || failure "IDF_TARGET not set in CMakeCache.txt after fullclean and reconfigure" + rm sdkconfig.defaults + + print_status "IDF_TARGET takes precedence over the value of CONFIG_IDF_TARGET in sdkconfig.defaults" + clean_build_dir + rm sdkconfig + echo "CONFIG_IDF_TARGET=\"${other_target}\"" > sdkconfig.defaults + export IDF_TARGET=esp32 + idf.py reconfigure || failure "Failed to reconfigure with default target set in sdkconfig.defaults and different IDF_TARGET in the environment" + grep "CONFIG_IDF_TARGET=\"esp32\"" sdkconfig || failure "Didn't find the expected CONFIG_IDF_TARGET value" + grep "CONFIG_IDF_TARGET_ESP32=y" sdkconfig || failure "Didn't find CONFIG_IDF_TARGET_${other_target^^} value" + grep "IDF_TARGET:STRING=esp32" build/CMakeCache.txt || failure "IDF_TARGET not set in CMakeCache.txt after fullclean and reconfigure" + rm sdkconfig.defaults + unset IDF_TARGET + unset other_target # done changing target from the default clean_build_dir rm sdkconfig diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index bca13ecb0..6a9b058a0 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -158,6 +158,18 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False): if not os.path.isdir(build_dir): os.makedirs(build_dir) cache_path = os.path.join(build_dir, "CMakeCache.txt") + if not os.path.exists(cache_path) and not os.environ.get("IDF_TARGET"): + # CMakeCache.txt does not exist yet, and IDF_TARGET is not set in the environment. + # Try to guess the target from sdkconfig, so that it doesn't get reset to the default ("esp32"). + sdkconfig_path = os.path.join(args.project_dir, "sdkconfig") + # Also consider the CONFIG_IDF_TARGET value in sdkconfig.defaults. + sdkconfig_defaults_path = os.path.join(args.project_dir, "sdkconfig.defaults") + idf_target_from_sdkconfig = (get_sdkconfig_value(sdkconfig_path, "CONFIG_IDF_TARGET") or + get_sdkconfig_value(sdkconfig_defaults_path, "CONFIG_IDF_TARGET")) + if idf_target_from_sdkconfig: + if args.verbose: + print("IDF_TARGET is not set, guessed '%s' from sdkconfig" % (idf_target_from_sdkconfig)) + args.define_cache_entry.append("IDF_TARGET=" + idf_target_from_sdkconfig) args.define_cache_entry.append("CCACHE_ENABLE=%d" % args.ccache) @@ -220,3 +232,23 @@ def merge_action_lists(*action_lists): merged_actions["actions"].update(action_list.get("actions", {})) merged_actions["global_action_callbacks"].extend(action_list.get("global_action_callbacks", [])) return merged_actions + + +def get_sdkconfig_value(sdkconfig_file, key): + """ + Return the value of given key from sdkconfig_file. + If sdkconfig_file does not exist or the option is not present, returns None. + """ + assert key.startswith("CONFIG_") + if not os.path.exists(sdkconfig_file): + return None + # keep track of the last seen value for the given key + value = None + # if the value is quoted, this excludes the quotes from the value + pattern = re.compile(r"^{}=\"?([^\"]*)\"?$".format(key)) + with open(sdkconfig_file, "r") as f: + for line in f: + match = re.match(pattern, line) + if match: + value = match.group(1) + return value From 1adf83757767e381fb6d08099a5e97b5d7127275 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Thu, 30 Jan 2020 19:09:59 +0100 Subject: [PATCH 2/4] docs: update build system guide on setting IDF_TARGET --- docs/en/api-guides/build-system.rst | 32 +++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/en/api-guides/build-system.rst b/docs/en/api-guides/build-system.rst index de2a6479f..e804024bf 100644 --- a/docs/en/api-guides/build-system.rst +++ b/docs/en/api-guides/build-system.rst @@ -41,7 +41,7 @@ Concepts - "components" are modular pieces of standalone code which are compiled into static libraries (.a files) and linked into an app. Some are provided by ESP-IDF itself, others may be sourced from other places. -- "Target" is the hardware for which an application is built. At the moment, ESP-IDF supports only one target, ``esp32``. +- "Target" is the hardware for which an application is built. At the moment, ESP-IDF supports ``esp32`` and ``esp32s2beta`` targets. Some things are not part of the project: @@ -69,6 +69,7 @@ The :ref:`getting started guide ` contains a brief introd Type ``idf.py --help`` for a list of commands. Here are a summary of the most useful ones: +- ``idf.py set-target `` sets the target (chip) for which the project is built. See :ref:`selecting-idf-target`. - ``idf.py menuconfig`` runs the "menuconfig" tool to configure the project. - ``idf.py build`` will build the project found in the current directory. This can involve multiple steps: @@ -946,14 +947,37 @@ The bootloader is a special "subproject" inside :idf:`/components/bootloader/sub The subproject is inserted as an external project from the top-level project, by the file :idf_file:`/components/bootloader/project_include.cmake`. The main build process runs CMake for the subproject, which includes discovering components (a subset of the main components) and generating a bootloader-specific config (derived from the main ``sdkconfig``). +.. _selecting-idf-target: + Selecting the Target ==================== -Currently ESP-IDF supports one target, ``esp32``. It is used by default by the build system. Developers working on adding multiple target support can change the target as follows:: +ESP-IDF supports multiple targets (chips). The identifiers used for each chip are as follows: - rm sdkconfig - idf.py -DIDF_TARGET=new_target reconfigure +* ``esp32`` — for ESP32-D0WD, ESP32-D2WD, ESP32-S0WD (ESP-SOLO), ESP32-U4WD, ESP32-PICO-D4 +* ``esp32s2beta``— for ESP32-S2-beta (engineering samples) +To select the target before building the project, use ``idf.py set-target `` command, for example:: + + idf.py set-target esp32s2beta + +.. important:: + + ``idf.py set-target`` will clear the build directory and re-generate the ``sdkconfig`` file from scratch. The old ``sdkconfig`` file will be saved as ``sdkconfig.old``. + +.. note:: + + The behavior of ``idf.py set-target`` command is equivalent to: + + 1. clearing the build directory (``idf.py fullclean``) + 2. removing the sdkconfig file (``mv sdkconfig sdkconfig.old``) + 3. configuring the project with the new target (``idf.py -DIDF_TARGET=esp32 reconfigure``) + +It is also possible to pass the desired ``IDF_TARGET`` as an environement variable (e.g. ``export IDF_TARGET=esp32s2beta``) or as a CMake variable (e.g. ``-DIDF_TARGET=esp32s2beta`` argument to CMake or idf.py). Setting the environment variable is a convenient method if you mostly work with one type of the chip. + +To specify the _default_ value of ``IDF_TARGET`` for a given project, add ``CONFIG_IDF_TARGET`` value to ``sdkconfig.defaults``. For example, ``CONFIG_IDF_TARGET="esp32s2beta"``. This value will be used if ``IDF_TARGET`` is not specified by other method: using an environment variable, CMake variable, or ``idf.py set-target`` command. + +If the target has not been set by any of these methods, the build system will default to ``esp32`` target. Writing Pure CMake Components ============================= From 6ae3311f48398ec2406e30c9849ff2cbc647c2e5 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 31 Jan 2020 11:08:22 +0100 Subject: [PATCH 3/4] idf.py: fail and show instructions on IDF_TARGET mismatch Closes IDF-869 --- tools/ci/test_build_system_cmake.sh | 16 ++++++- tools/idf_py_actions/tools.py | 67 +++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/tools/ci/test_build_system_cmake.sh b/tools/ci/test_build_system_cmake.sh index a8d3adf94..73c140503 100755 --- a/tools/ci/test_build_system_cmake.sh +++ b/tools/ci/test_build_system_cmake.sh @@ -292,7 +292,7 @@ function run_tests() rm sdkconfig rm sdkconfig.defaults - # the next four tests use the esp32s2beta target + # the next tests use the esp32s2beta target export other_target=esp32s2beta print_status "Can override IDF_TARGET from environment" @@ -353,6 +353,20 @@ function run_tests() rm sdkconfig.defaults unset IDF_TARGET + print_status "idf.py fails if IDF_TARGET settings don't match in sdkconfig, CMakeCache.txt, and the environment" + clean_build_dir + rm sdkconfig + idf.py set-target ${other_target} || failure "Couldn't set target to ${other_target}" + # Change to a different IDF_TARGET in the environment + export IDF_TARGET=esp32 + ! idf.py reconfigure || failure "Build did't fail when IDF_TARGET was set to an incompatible value in the environment" + # Now make sdkconfig consistent with the environement (note: not really consistent, just for the purpose of the test) + echo "CONFIG_IDF_TARGET=\"esp32\"" >> sdkconfig + ! idf.py reconfigure || failure "Build did't fail when IDF_TARGET in CMakeCache.txt didn't match the environment" + # Now unset IDF_TARGET in the environment, sdkconfig and CMakeCache.txt are still inconsistent + unset IDF_TARGET + ! idf.py reconfigure || failure "Build did't fail when IDF_TARGET in CMakeCache.txt didn't match the sdkconfig" + unset other_target # done changing target from the default clean_build_dir rm sdkconfig diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 6a9b058a0..ca1e6cb02 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -157,19 +157,13 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False): build_dir = args.build_dir if not os.path.isdir(build_dir): os.makedirs(build_dir) + + # Parse CMakeCache, if it exists cache_path = os.path.join(build_dir, "CMakeCache.txt") - if not os.path.exists(cache_path) and not os.environ.get("IDF_TARGET"): - # CMakeCache.txt does not exist yet, and IDF_TARGET is not set in the environment. - # Try to guess the target from sdkconfig, so that it doesn't get reset to the default ("esp32"). - sdkconfig_path = os.path.join(args.project_dir, "sdkconfig") - # Also consider the CONFIG_IDF_TARGET value in sdkconfig.defaults. - sdkconfig_defaults_path = os.path.join(args.project_dir, "sdkconfig.defaults") - idf_target_from_sdkconfig = (get_sdkconfig_value(sdkconfig_path, "CONFIG_IDF_TARGET") or - get_sdkconfig_value(sdkconfig_defaults_path, "CONFIG_IDF_TARGET")) - if idf_target_from_sdkconfig: - if args.verbose: - print("IDF_TARGET is not set, guessed '%s' from sdkconfig" % (idf_target_from_sdkconfig)) - args.define_cache_entry.append("IDF_TARGET=" + idf_target_from_sdkconfig) + cache = _parse_cmakecache(cache_path) if os.path.exists(cache_path) else {} + + # Validate or set IDF_TARGET + _guess_or_check_idf_target(args, prog_name, cache) args.define_cache_entry.append("CCACHE_ENABLE=%d" % args.ccache) @@ -199,8 +193,6 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False): os.remove(cache_path) raise - # Learn some things from the CMakeCache.txt file in the build directory - cache = _parse_cmakecache(cache_path) try: generator = cache["CMAKE_GENERATOR"] except KeyError: @@ -252,3 +244,50 @@ def get_sdkconfig_value(sdkconfig_file, key): if match: value = match.group(1) return value + + +def _guess_or_check_idf_target(args, prog_name, cache): + """ + If CMakeCache.txt doesn't exist, and IDF_TARGET is not set in the environment, guess the value from + sdkconfig or sdkconfig.defaults, and pass it to CMake in IDF_TARGET variable. + + Otherwise, cross-check the three settings (sdkconfig, CMakeCache, environment) and if there is + mismatch, fail with instructions on how to fix this. + """ + # Default locations of sdkconfig files. + # FIXME: they may be overridden in the project or by a CMake variable (IDF-1369). + sdkconfig_path = os.path.join(args.project_dir, "sdkconfig") + sdkconfig_defaults_path = os.path.join(args.project_dir, "sdkconfig.defaults") + + # These are used to guess the target from sdkconfig, or set the default target by sdkconfig.defaults. + idf_target_from_sdkconfig = get_sdkconfig_value(sdkconfig_path, "CONFIG_IDF_TARGET") + idf_target_from_sdkconfig_defaults = get_sdkconfig_value(sdkconfig_defaults_path, "CONFIG_IDF_TARGET") + idf_target_from_env = os.environ.get("IDF_TARGET") + idf_target_from_cache = cache.get("IDF_TARGET") + + if not cache and not idf_target_from_env: + # CMakeCache.txt does not exist yet, and IDF_TARGET is not set in the environment. + guessed_target = idf_target_from_sdkconfig or idf_target_from_sdkconfig_defaults + if guessed_target: + if args.verbose: + print("IDF_TARGET is not set, guessed '%s' from sdkconfig" % (guessed_target)) + args.define_cache_entry.append("IDF_TARGET=" + guessed_target) + + elif idf_target_from_env: + # Let's check that IDF_TARGET values are consistent + if idf_target_from_sdkconfig and idf_target_from_sdkconfig != idf_target_from_env: + raise FatalError("Project sdkconfig was generated for target '{t_conf}', but environment variable IDF_TARGET " + "is set to '{t_env}'. Run '{prog} set-target {t_env}' to generate new sdkconfig file for target {t_env}." + .format(t_conf=idf_target_from_sdkconfig, t_env=idf_target_from_env, prog=prog_name)) + + if idf_target_from_cache and idf_target_from_cache != idf_target_from_env: + raise FatalError("Target settings are not consistent: '{t_env}' in the environment, '{t_cache}' in CMakeCache.txt. " + "Run '{prog} fullclean' to start again." + .format(t_env=idf_target_from_env, t_cache=idf_target_from_cache, prog=prog_name)) + + elif idf_target_from_cache and idf_target_from_sdkconfig and idf_target_from_cache != idf_target_from_sdkconfig: + # This shouldn't happen, unless the user manually edits CMakeCache.txt or sdkconfig, but let's check anyway. + raise FatalError("Project sdkconfig was generated for target '{t_conf}', but CMakeCache.txt contains '{t_cache}'. " + "To keep the setting in sdkconfig ({t_conf}) and re-generate CMakeCache.txt, run '{prog} fullclean'. " + "To re-generate sdkconfig for '{t_cache}' target, run '{prog} set-target {t_cache}'." + .format(t_conf=idf_target_from_sdkconfig, t_cache=idf_target_from_cache, prog=prog_name)) From e5473299352229f84f66b12a63787ef7be1a077a Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 5 Feb 2020 15:27:59 +0100 Subject: [PATCH 4/4] ci: fix build system test on macOS Replaces ${var^^} expansion supported with Bash >= 4 with 'tr' --- tools/ci/test_build_system_cmake.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci/test_build_system_cmake.sh b/tools/ci/test_build_system_cmake.sh index 73c140503..abc4a5a55 100755 --- a/tools/ci/test_build_system_cmake.sh +++ b/tools/ci/test_build_system_cmake.sh @@ -337,7 +337,8 @@ function run_tests() echo "CONFIG_IDF_TARGET=\"${other_target}\"" > sdkconfig.defaults idf.py reconfigure || failure "Failed to reconfigure with default target set in sdkconfig.defaults" grep "CONFIG_IDF_TARGET=\"${other_target}\"" sdkconfig || failure "Didn't find the expected CONFIG_IDF_TARGET value" - grep "CONFIG_IDF_TARGET_${other_target^^}=y" sdkconfig || failure "Didn't find CONFIG_IDF_TARGET_${other_target^^} value" + other_target_caps=$(tr 'a-z' 'A-Z' <<< "${other_target}") + grep "CONFIG_IDF_TARGET_${other_target_caps}=y" sdkconfig || failure "Didn't find CONFIG_IDF_TARGET_${other_target_caps} value" grep "IDF_TARGET:STRING=${other_target}" build/CMakeCache.txt || failure "IDF_TARGET not set in CMakeCache.txt after fullclean and reconfigure" rm sdkconfig.defaults @@ -348,7 +349,7 @@ function run_tests() export IDF_TARGET=esp32 idf.py reconfigure || failure "Failed to reconfigure with default target set in sdkconfig.defaults and different IDF_TARGET in the environment" grep "CONFIG_IDF_TARGET=\"esp32\"" sdkconfig || failure "Didn't find the expected CONFIG_IDF_TARGET value" - grep "CONFIG_IDF_TARGET_ESP32=y" sdkconfig || failure "Didn't find CONFIG_IDF_TARGET_${other_target^^} value" + grep "CONFIG_IDF_TARGET_ESP32=y" sdkconfig || failure "Didn't find CONFIG_IDF_TARGET_ESP32 value" grep "IDF_TARGET:STRING=esp32" build/CMakeCache.txt || failure "IDF_TARGET not set in CMakeCache.txt after fullclean and reconfigure" rm sdkconfig.defaults unset IDF_TARGET