Merge branch 'feature/idf_tools_dir_2' into 'master'

IDF tools metadata file and IDF_TOOLS_PATH support

See merge request idf/esp-idf!4411

.gitlab-ci.yml

@@ -601,6 +601,14 @@ test_idf_size:
     - cd ${IDF_PATH}/tools/test_idf_size
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
 
+test_idf_tools:
+  <<: *host_test_template
+  script:
+    # Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment
+    - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/opt/espressif${CUSTOM_TOOLCHAIN_PATH:+\|${CUSTOM_TOOLCHAIN_PATH}}" | tr "\n" ":"); echo ${p%:})
+    - cd ${IDF_PATH}/tools/test_idf_tools
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
+
 test_esp_err_to_name_on_host:
   <<: *host_test_template
   artifacts:

tools/check_python_dependencies.py

@@ -64,7 +64,15 @@ if __name__ == "__main__":
         print('The following Python requirements are not satisfied:')
         for requirement in not_satisfied:
             print(requirement)
-        if sys.platform == "win32" and os.environ.get("MSYSTEM", None) == "MINGW32" and "/mingw32/bin/python" in sys.executable:
+        if os.environ.get('IDF_PYTHON_ENV_PATH'):
+            # We are running inside a private virtual environment under IDF_TOOLS_PATH,
+            # ask the user to run install.bat again.
+            if sys.platform == "win32" and not os.environ.get("MSYSTEM"):
+                install_script = 'install.bat'
+            else:
+                install_script = 'install.sh'
+            print('To install the missing packages, please run "%s"' % os.path.join(idf_path, install_script))
+        elif sys.platform == "win32" and os.environ.get("MSYSTEM", None) == "MINGW32" and "/mingw32/bin/python" in sys.executable:
             print("The recommended way to install a packages is via \"pacman\". Please run \"pacman -Ss <package_name>\" for"
                   " searching the package database and if found then "
                   "\"pacman -S mingw-w64-i686-python{}-<package_name>\" for installing it.".format(sys.version_info[0],))

tools/ci/check-executable.sh

@@ -2,9 +2,16 @@
 # This script finds executable files in the repository, excluding some directories,
 # then prints the list of all files which are not in executable-list.txt.
 # Returns with error if this list is non-empty.
+# Also checks if executable-list.txt is sorted and has no duplicates.
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+set -o nounset # Exit if variable not set.
+
 
 cd $IDF_PATH
 
+in_list=tools/ci/executable-list.txt
 tmp_list=$(mktemp)
 out_list=$(mktemp)
 
@@ -27,7 +34,7 @@ find . -type d \( \
     | sed "s|^\./||" > $tmp_list
 
 # this looks for lines present in tmp_list but not in executable-list.txt
-comm -13 <(cat tools/ci/executable-list.txt | sed -n "/^#/!p" | sort) <(sort $tmp_list) > $out_list
+comm -13 <(cat $in_list | sed -n "/^#/!p" | sort) <(sort $tmp_list) > $out_list
 
 ret=0
 if [ -s $out_list ]; then
@@ -42,6 +49,17 @@ if [ -s $out_list ]; then
     echo ""
 fi
 
+if ! diff <(cat $in_list | sed -n "/^#/!p" | sort | uniq) $in_list; then
+    echo "$in_list is not sorted or has duplicate entries"
+    ret=2
+fi
+
+for filename in $(cat $in_list | sed -n "/^#/!p"); do
+    if [ ! -f "$filename" ]; then
+        echo "Warning: file '$filename' is present in '$in_list', but does not exist"
+    fi
+done
+
 rm $tmp_list
 rm $out_list
 

tools/ci/executable-list.txt

@@ -1,39 +1,49 @@
-# Some tools
+components/app_update/otatool.py
-components/app_update/gen_empty_partition.py
+components/efuse/efuse_table_gen.py
-tools/elf_to_ld.sh
+components/efuse/test_efuse_host/efuse_tests.py
 components/espcoredump/espcoredump.py
+components/espcoredump/test/test_espcoredump.py
+components/espcoredump/test/test_espcoredump.sh
 components/heap/test_multi_heap_host/test_all_configs.sh
 components/idf_test/unit_test/TestCaseScript/IDFUnitTest/__init__.py
 components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
-components/efuse/efuse_table_gen.py
-components/efuse/test_efuse_host/efuse_tests.py
 components/partition_table/gen_esp32part.py
 components/partition_table/parttool.py
-components/app_update/gen_empty_partition.py
-components/app_update/otatool.py
 components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py
-components/ulp/esp32ulp_mapgen.py
 components/spiffs/spiffsgen.py
+components/ulp/esp32ulp_mapgen.py
 docs/check_doc_warnings.sh
 docs/check_lang_folder_sync.sh
-docs/gen-kconfig-doc.py
+docs/gen-dxd.py
 docs/gen-version-specific-includes.py
+examples/build_system/cmake/idf_as_lib/build-esp32.sh
+examples/build_system/cmake/idf_as_lib/build.sh
+examples/build_system/cmake/idf_as_lib/run-esp32.sh
+examples/build_system/cmake/idf_as_lib/run.sh
+examples/storage/parttool/parttool_example.py
+examples/system/ota/otatool/otatool_example.py
+tools/check_kconfigs.py
+tools/check_python_dependencies.py
 tools/ci/apply_bot_filter.py
 tools/ci/build_examples.sh
 tools/ci/build_examples_cmake.sh
 tools/ci/check-executable.sh
 tools/ci/check-line-endings.sh
+tools/ci/check_examples_cmake_make.sh
+tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py
 tools/ci/envsubst.py
 tools/ci/get-full-sources.sh
 tools/ci/mirror-submodule-update.sh
 tools/ci/mirror-synchronize.sh
+tools/ci/multirun_with_pyenv.sh
 tools/ci/push_to_github.sh
 tools/ci/test_build_system.sh
 tools/ci/test_build_system_cmake.sh
 tools/ci/test_configure_ci_environment.sh
 tools/cmake/convert_to_cmake.py
 tools/cmake/run_cmake_lint.sh
+tools/elf_to_ld.sh
 tools/esp_app_trace/logtrace_proc.py
 tools/esp_app_trace/sysviewtrace_proc.py
 tools/esp_app_trace/test/logtrace/test.sh
@@ -43,41 +53,22 @@ tools/gen_esp_err_to_name.py
 tools/idf.py
 tools/idf_monitor.py
 tools/idf_size.py
+tools/idf_tools.py
 tools/kconfig/check.sh
-tools/kconfig/conf
 tools/kconfig/lxdialog/check-lxdialog.sh
-tools/kconfig/mconf
 tools/kconfig/merge_config.sh
 tools/kconfig/streamline_config.pl
-tools/mass_mfg/mfg_gen.py
-tools/test_idf_monitor/run_test_idf_monitor.py
-tools/unit-test-app/unit_test.py
-tools/windows/eclipse_make.sh
-tools/ci/build_examples_cmake.sh
-tools/ci/test_build_system_cmake.sh
-tools/ci/check_examples_cmake_make.sh
-tools/ci/check_ut_cmake_make.sh
-tools/cmake/convert_to_cmake.py
-tools/cmake/run_cmake_lint.sh
-tools/idf.py
 tools/kconfig_new/confgen.py
 tools/kconfig_new/confserver.py
 tools/kconfig_new/test/test_confserver.py
-tools/windows/tool_setup/build_installer.sh
-tools/test_idf_size/test.sh
-tools/check_python_dependencies.py
-docs/gen-dxd.py
-tools/ci/multirun_with_pyenv.sh
-components/espcoredump/test/test_espcoredump.py
-components/espcoredump/test/test_espcoredump.sh
 tools/ldgen/ldgen.py
 tools/ldgen/test/test_fragments.py
 tools/ldgen/test/test_generation.py
-examples/build_system/cmake/idf_as_lib/build-esp32.sh
+tools/mass_mfg/mfg_gen.py
-examples/build_system/cmake/idf_as_lib/build.sh
-examples/build_system/cmake/idf_as_lib/run-esp32.sh
-examples/build_system/cmake/idf_as_lib/run.sh
-examples/storage/parttool/parttool_example.py
-examples/system/ota/otatool/otatool_example.py
-tools/check_kconfigs.py
 tools/test_check_kconfigs.py
+tools/test_idf_monitor/run_test_idf_monitor.py
+tools/test_idf_size/test.sh
+tools/test_idf_tools/test_idf_tools.py
+tools/unit-test-app/unit_test.py
+tools/windows/eclipse_make.sh
+tools/windows/tool_setup/build_installer.sh

tools/idf.py

@@ -51,6 +51,10 @@ PYTHON = sys.executable
 # you have to pass env=os.environ explicitly anywhere that we create a process
 os.environ["PYTHON"] = sys.executable
 
+# Name of the program, normally 'idf.py'.
+# Can be overridden from idf.bat using IDF_PY_PROGRAM_NAME
+PROG = os.getenv('IDF_PY_PROGRAM_NAME', sys.argv[0])
+
 # Make flavors, across the various kinds of Windows environments & POSIX...
 if "MSYSTEM" in os.environ:  # MSYS
     MAKE_CMD = "make"
@@ -105,15 +109,15 @@ def check_environment():
     (cmake will check a lot of other things)
     """
     if not executable_exists(["cmake", "--version"]):
-        raise FatalError("'cmake' must be available on the PATH to use idf.py")
+        raise FatalError("'cmake' must be available on the PATH to use %s" % PROG)
     # find the directory idf.py is in, then the parent directory of this, and assume this is IDF_PATH
     detected_idf_path = _realpath(os.path.join(os.path.dirname(__file__), ".."))
     if "IDF_PATH" in os.environ:
         set_idf_path = _realpath(os.environ["IDF_PATH"])
         if set_idf_path != detected_idf_path:
-            print("WARNING: IDF_PATH environment variable is set to %s but idf.py path indicates IDF directory %s. "
+            print("WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. "
                   "Using the environment variable directory, but results may be unexpected..."
-                  % (set_idf_path, detected_idf_path))
+                  % (set_idf_path, PROG, detected_idf_path))
     else:
         print("Setting IDF_PATH environment variable: %s" % detected_idf_path)
         os.environ["IDF_PATH"] = detected_idf_path
@@ -143,7 +147,7 @@ def detect_cmake_generator():
     for (generator, _, version_check, _) in GENERATORS:
         if executable_exists(version_check):
             return generator
-    raise FatalError("To use idf.py, either the 'ninja' or 'GNU make' build tool must be available in the PATH")
+    raise FatalError("To use %s, either the 'ninja' or 'GNU make' build tool must be available in the PATH" % PROG)
 
 
 def _ensure_build_directory(args, always_run_cmake=False):
@@ -160,9 +164,9 @@ def _ensure_build_directory(args, always_run_cmake=False):
     # Verify the project directory
     if not os.path.isdir(project_dir):
         if not os.path.exists(project_dir):
-            raise FatalError("Project directory %s does not exist")
+            raise FatalError("Project directory %s does not exist" % project_dir)
         else:
-            raise FatalError("%s must be a project directory")
+            raise FatalError("%s must be a project directory" % project_dir)
     if not os.path.exists(os.path.join(project_dir, "CMakeLists.txt")):
         raise FatalError("CMakeLists.txt not found in project directory %s" % project_dir)
 
@@ -201,14 +205,14 @@ def _ensure_build_directory(args, always_run_cmake=False):
     if args.generator is None:
         args.generator = generator  # reuse the previously configured generator, if none was given
     if generator != args.generator:
-        raise FatalError("Build is configured for generator '%s' not '%s'. Run 'idf.py fullclean' to start again."
+        raise FatalError("Build is configured for generator '%s' not '%s'. Run '%s fullclean' to start again."
-                         % (generator, args.generator))
+                         % (generator, args.generator, PROG))
 
     try:
         home_dir = cache["CMAKE_HOME_DIRECTORY"]
         if _realpath(home_dir) != _realpath(project_dir):
-            raise FatalError("Build directory '%s' configured for project '%s' not '%s'. Run 'idf.py fullclean' to start again."
+            raise FatalError("Build directory '%s' configured for project '%s' not '%s'. Run '%s fullclean' to start again."
-                             % (build_dir, _realpath(home_dir), _realpath(project_dir)))
+                             % (build_dir, _realpath(home_dir), _realpath(project_dir), PROG))
     except KeyError:
         pass  # if cmake failed part way, CMAKE_HOME_DIRECTORY may not be set yet
 
@@ -310,7 +314,7 @@ def monitor(action, args):
     if not os.path.exists(elf_file):
         raise FatalError("ELF file '%s' not found. You need to build & flash the project before running 'monitor', "
                          "and the binary on the device must match the one in the build directory exactly. "
-                         "Try 'idf.py flash monitor'." % elf_file)
+                         "Try '%s flash monitor'." % (elf_file, PROG))
     idf_monitor = os.path.join(os.environ["IDF_PATH"], "tools/idf_monitor.py")
     monitor_args = [PYTHON, idf_monitor]
     if args.port is not None:
@@ -440,7 +444,7 @@ def print_closing_message(args):
             args.baud,
             flasher_args["extra_esptool_args"]["after"],
             cmd.strip()))
-        print("or run 'idf.py -p %s %s'" % (args.port or "(PORT)", key + "-flash" if key != "project" else "flash",))
+        print("or run '%s -p %s %s'" % (PROG, args.port or "(PORT)", key + "-flash" if key != "project" else "flash",))
 
     if "all" in args.actions or "build" in args.actions:
         print_flashing_message("Project", "project")
@@ -543,7 +547,7 @@ def main():
     except NameError:
         pass
 
-    parser = argparse.ArgumentParser(description='ESP-IDF build management tool')
+    parser = argparse.ArgumentParser(prog=PROG, description='ESP-IDF build management tool')
     parser.add_argument('-p', '--port', help="Serial port",
                         default=os.environ.get('ESPPORT', None))
     parser.add_argument('-b', '--baud', help="Baud rate",

tools/test_idf_tools/test_idf_tools.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+#
+# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import unittest
+import tempfile
+import shutil
+
+try:
+    from contextlib import redirect_stdout
+except ImportError:
+    import contextlib
+
+    @contextlib.contextmanager
+    def redirect_stdout(target):
+        original = sys.stdout
+        sys.stdout = target
+        yield
+        sys.stdout = original
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+# Need to do this before importing idf_tools.py
+os.environ['IDF_MAINTAINER'] = '1'
+
+try:
+    import idf_tools
+except ImportError:
+    sys.path.append('..')
+    import idf_tools
+
+
+class TestUsage(unittest.TestCase):
+
+    def test_usage_basic(self):
+        old_tools_dir = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT)
+
+        mirror_prefix_map = None
+        if os.path.exists(old_tools_dir):
+            mirror_prefix_map = 'https://dl.espressif.com/dl,file://' + os.path.join(old_tools_dir, 'dist')
+            mirror_prefix_map += ';https://github.com/espressif/.*/releases/download/.*/,file://' + os.path.join(old_tools_dir, 'dist', '')
+        if mirror_prefix_map:
+            print('Using IDF_MIRROR_PREFIX_MAP={}'.format(mirror_prefix_map))
+            os.environ['IDF_MIRROR_PREFIX_MAP'] = mirror_prefix_map
+
+        temp_tools_dir = tempfile.mkdtemp(prefix='idf_tools_tmp')
+        print('Using IDF_TOOLS_PATH={}'.format(temp_tools_dir))
+        os.environ['IDF_TOOLS_PATH'] = temp_tools_dir
+
+        self.addCleanup(shutil.rmtree, temp_tools_dir)
+
+        output_stream = StringIO()
+        with redirect_stdout(output_stream):
+            idf_tools.main(['list'])
+        output = output_stream.getvalue()
+
+        xtensa_esp32_elf_version = '1.22.0-80-g6c4433a-5.2.0'
+        esp32ulp_version = '2.28.51.20170517'
+
+        self.assertIn('* xtensa-esp32-elf:', output)
+        self.assertIn('- %s (recommended)' % xtensa_esp32_elf_version, output)
+        self.assertIn('* esp32ulp-elf', output)
+        self.assertIn('- %s (recommended)' % esp32ulp_version, output)
+
+        output_stream = StringIO()
+        with redirect_stdout(output_stream):
+            idf_tools.main(['install'])
+        output = output_stream.getvalue()
+
+        self.assertIn('Installing esp32ulp-elf@' + esp32ulp_version, output)
+        self.assertIn('Downloading binutils-esp32ulp', output)
+        self.assertIn('Installing xtensa-esp32-elf@' + xtensa_esp32_elf_version, output)
+        self.assertIn('Downloading xtensa-esp32-elf', output)
+        self.assertIn('to ' + os.path.join(temp_tools_dir, 'dist'), output)
+
+        output_stream = StringIO()
+        with redirect_stdout(output_stream):
+            idf_tools.main(['check'])
+        output = output_stream.getvalue()
+
+        self.assertIn('version installed in tools directory: ' + esp32ulp_version, output)
+        self.assertIn('version installed in tools directory: ' + xtensa_esp32_elf_version, output)
+
+        output_stream = StringIO()
+        with redirect_stdout(output_stream):
+            idf_tools.main(['export'])
+        output = output_stream.getvalue()
+
+        self.assertIn('%s/tools/esp32ulp-elf/%s/esp32ulp-elf-binutils/bin' %
+                      (temp_tools_dir, esp32ulp_version), output)
+        self.assertIn('%s/tools/xtensa-esp32-elf/%s/xtensa-esp32-elf/bin' %
+                      (temp_tools_dir, xtensa_esp32_elf_version), output)
+
+
+class TestMaintainer(unittest.TestCase):
+
+    def test_validation(self):
+        idf_tools.main(['validate'])
+
+    def test_json_rewrite(self):
+        idf_tools.main(['rewrite'])
+        idf_path = os.getenv('IDF_PATH')
+        if not idf_path:
+            self.fail('IDF_PATH needs to be set to run this test')
+        with open(os.path.join(idf_path, 'tools/tools.json'), 'r') as f:
+            json_old = f.read()
+        with open(os.path.join(idf_path, 'tools/tools.new.json'), 'r') as f:
+            json_new = f.read()
+        self.assertEqual(json_old, json_new)
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/tools.json

@@ -0,0 +1,347 @@
+{
+  "tools": [
+    {
+      "description": "Toolchain for Xtensa (ESP32) based on GCC",
+      "export_paths": [
+        [
+          "xtensa-esp32-elf",
+          "bin"
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/espressif/crosstool-NG",
+      "install": "always",
+      "license": "GPL-3.0-with-GCC-exception",
+      "name": "xtensa-esp32-elf",
+      "version_cmd": [
+        "xtensa-esp32-elf-gcc",
+        "--version"
+      ],
+      "version_regex": "\\(crosstool-NG\\s+(?:crosstool-ng-)?([0-9a-z\\.\\-]+)\\)\\s*([0-9\\.]+)",
+      "version_regex_replace": "\\1-\\2",
+      "versions": [
+        {
+          "name": "1.22.0-80-g6c4433a5-5.2.0",
+          "status": "recommended",
+          "win32": {
+            "sha256": "f217fccbeaaa8c92db239036e0d6202458de4488b954a3a38f35ac2ec48058a4",
+            "size": 125719261,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
+          },
+          "win64": {
+            "sha256": "f217fccbeaaa8c92db239036e0d6202458de4488b954a3a38f35ac2ec48058a4",
+            "size": 125719261,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
+          }
+        },
+        {
+          "linux-amd64": {
+            "sha256": "3fe96c151d46c1d4e5edc6ed690851b8e53634041114bad04729bc16b0445156",
+            "size": 44219107,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22.0-80-g6c4433a-5.2.0.tar.gz"
+          },
+          "linux-i686": {
+            "sha256": "b4055695ffc2dfc0bcb6dafdc2572a6e01151c4179ef5fa972b3fcb2183eb155",
+            "size": 45566336,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-linux32-1.22.0-80-g6c4433a-5.2.0.tar.gz"
+          },
+          "macos": {
+            "sha256": "a4307a97945d2f2f2745f415fbe80d727750e19f91f9a1e7e2f8a6065652f9da",
+            "size": 46517409,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-osx-1.22.0-80-g6c4433a-5.2.0.tar.gz"
+          },
+          "name": "1.22.0-80-g6c4433a-5.2.0",
+          "status": "recommended"
+        }
+      ]
+    },
+    {
+      "description": "Toolchain for ESP32 ULP coprocessor",
+      "export_paths": [
+        [
+          "esp32ulp-elf-binutils",
+          "bin"
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/espressif/binutils-esp32ulp",
+      "install": "always",
+      "license": "GPL-2.0-or-later",
+      "name": "esp32ulp-elf",
+      "version_cmd": [
+        "esp32ulp-elf-as",
+        "--version"
+      ],
+      "version_regex": "\\(GNU Binutils\\)\\s+([0-9a-z\\.\\-]+)",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "c1bbcd65e1e30c7312a50344c8dbc70c2941580a79aa8f8abbce8e0e90c79566",
+            "size": 8246604,
+            "url": "https://dl.espressif.com/dl/binutils-esp32ulp-linux64-2.28.51-esp32ulp-20180809.tar.gz"
+          },
+          "macos": {
+            "sha256": "c92937d85cc9a90eb6c6099ce767ca021108c18c94e34bd7b1fa0cde168f94a0",
+            "size": 5726662,
+            "url": "https://dl.espressif.com/dl/binutils-esp32ulp-macos-2.28.51-esp32ulp-20180809.tar.gz"
+          },
+          "name": "2.28.51.20170517",
+          "status": "recommended",
+          "win32": {
+            "sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd",
+            "size": 12231559,
+            "url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip"
+          },
+          "win64": {
+            "sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd",
+            "size": 12231559,
+            "url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "CMake build system",
+      "export_paths": [
+        [
+          "bin"
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/Kitware/CMake",
+      "install": "on_request",
+      "license": "BSD-3-Clause",
+      "name": "cmake",
+      "platform_overrides": [
+        {
+          "install": "always",
+          "platforms": [
+            "win32",
+            "win64"
+          ]
+        },
+        {
+          "export_paths": [
+            [
+              "CMake.app",
+              "Contents",
+              "bin"
+            ]
+          ],
+          "platforms": [
+            "macos"
+          ]
+        }
+      ],
+      "strip_container_dirs": 1,
+      "version_cmd": [
+        "cmake",
+        "--version"
+      ],
+      "version_regex": "cmake version ([0-9.]+)",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76",
+            "size": 38405896,
+            "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz"
+          },
+          "macos": {
+            "sha256": "fef537614d73fda848f6168273b6c7ba45f850484533361e7bc50ac1d315f780",
+            "size": 32062124,
+            "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Darwin-x86_64.tar.gz"
+          },
+          "name": "3.13.4",
+          "status": "recommended",
+          "win32": {
+            "sha256": "28daf772f55d817a13ef14e25af2a5569f8326dac66a6aa3cc5208cf1f8e943f",
+            "size": 26385104,
+            "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win32-x86.zip"
+          },
+          "win64": {
+            "sha256": "bcd477d49e4a9400b41213d53450b474beaedb264631693c958ef9affa8e5623",
+            "size": 29696565,
+            "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win64-x64.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "OpenOCD for ESP32",
+      "export_paths": [
+        [
+          "openocd-esp32",
+          "bin"
+        ]
+      ],
+      "export_vars": {
+        "OPENOCD_SCRIPTS": "${TOOL_PATH}/openocd-esp32/share/openocd/scripts"
+      },
+      "info_url": "https://github.com/espressif/openocd-esp32",
+      "install": "always",
+      "license": "GPL-2.0-only",
+      "name": "openocd-esp32",
+      "version_cmd": [
+        "openocd",
+        "--version"
+      ],
+      "version_regex": "Open On-Chip Debugger\\s+([a-z0-9.-]+)\\s+",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "e5b5579edffde090e426b4995b346e281843bf84394f8e68c8e41bd1e4c576bd",
+            "size": 1681596,
+            "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-linux64-0.10.0-esp32-20190313.tar.gz"
+          },
+          "macos": {
+            "sha256": "09504eea5aa92646a117f16573c95b34e04b4010791a2f8fefcd2bd8c430f081",
+            "size": 1760536,
+            "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-macos-0.10.0-esp32-20190313.tar.gz"
+          },
+          "name": "v0.10.0-esp32-20190313",
+          "status": "recommended",
+          "win32": {
+            "sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2",
+            "size": 2098513,
+            "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip"
+          },
+          "win64": {
+            "sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2",
+            "size": 2098513,
+            "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "menuconfig tool",
+      "export_paths": [
+        [
+          ""
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/espressif/kconfig-frontends",
+      "install": "never",
+      "license": "GPL-2.0-only",
+      "name": "mconf",
+      "platform_overrides": [
+        {
+          "install": "always",
+          "platforms": [
+            "win32",
+            "win64"
+          ]
+        }
+      ],
+      "strip_container_dirs": 1,
+      "version_cmd": [
+        "mconf-idf",
+        "-v"
+      ],
+      "version_regex": "mconf-idf version mconf-([a-z0-9.-]+)-win32",
+      "versions": [
+        {
+          "name": "v4.6.0.0-idf-20190313",
+          "status": "recommended",
+          "win32": {
+            "sha256": "051bef09c782bc31b737b047808d1b6588b0965101b77dc979af9139773c4b4f",
+            "size": 826167,
+            "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190313/mconf-v4.6.0.0-idf-20190313-win32.zip"
+          },
+          "win64": {
+            "sha256": "051bef09c782bc31b737b047808d1b6588b0965101b77dc979af9139773c4b4f",
+            "size": 826167,
+            "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190313/mconf-v4.6.0.0-idf-20190313-win32.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "Ninja build system",
+      "export_paths": [
+        [
+          ""
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/ninja-build/ninja",
+      "install": "on_request",
+      "license": "Apache-2.0",
+      "name": "ninja",
+      "platform_overrides": [
+        {
+          "install": "always",
+          "platforms": [
+            "win32",
+            "win64"
+          ]
+        }
+      ],
+      "version_cmd": [
+        "ninja",
+        "--version"
+      ],
+      "version_regex": "([0-9.]+)",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "978fd9e26c2db8d33392c6daef50e9edac0a3db6680710a9f9ad47e01f3e49b7",
+            "size": 85276,
+            "url": "https://dl.espressif.com/dl/ninja-1.9.0-linux64.tar.gz"
+          },
+          "macos": {
+            "sha256": "9504cd1783ef3c242d06330a50d54dc8f838b605f5fc3e892c47254929f7350c",
+            "size": 91457,
+            "url": "https://dl.espressif.com/dl/ninja-1.9.0-osx.tar.gz"
+          },
+          "name": "1.9.0",
+          "status": "recommended",
+          "win64": {
+            "sha256": "2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9",
+            "size": 254497,
+            "url": "https://dl.espressif.com/dl/ninja-1.9.0-win64.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "Ccache (compiler cache)",
+      "export_paths": [
+        [
+          ""
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/ccache/ccache",
+      "install": "never",
+      "license": "GPL-3.0-or-later",
+      "name": "ccache",
+      "platform_overrides": [
+        {
+          "install": "always",
+          "platforms": [
+            "win64"
+          ]
+        }
+      ],
+      "version_cmd": [
+        "ccache.exe",
+        "--version"
+      ],
+      "version_regex": "ccache version ([0-9.]+)",
+      "versions": [
+        {
+          "name": "3.7",
+          "status": "recommended",
+          "win64": {
+            "sha256": "37e833f3f354f1145503533e776c1bd44ec2e77ff8a2476a1d2039b0b10c78d6",
+            "size": 142401,
+            "url": "https://dl.espressif.com/dl/ccache-3.7-w64.zip"
+          }
+        }
+      ]
+    }
+  ],
+  "version": 1
+}

tools/tools_schema.json

@@ -0,0 +1,234 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/espressif/esp-idf/blob/master/tools/tools-schema.json",
+  "type": "object",
+  "properties": {
+    "version": {
+      "type": "integer",
+      "description": "Metadata file version"
+    },
+    "tools": {
+      "type": "array",
+      "description": "List of tools",
+      "items": {
+        "$ref": "#/definitions/toolInfo"
+      }
+    }
+  },
+  "required": [
+    "version",
+    "tools"
+  ],
+  "definitions": {
+    "toolInfo": {
+      "type": "object",
+      "description": "Information about one tool",
+      "properties": {
+        "name" : {
+          "description": "Tool name (used as a directory name)",
+          "type": "string"
+        },
+        "description" : {
+          "description": "A short (one sentence) description of the tool.",
+          "type": "string"
+        },
+        "export_paths": {
+          "$ref": "#/definitions/exportPaths"
+        },
+        "export_vars": {
+          "$ref": "#/definitions/envVars",
+          "description": "Some variable expansions are done on the values. 1) ${TOOL_PATH} is replaced with the directory where the tool is installed."
+        },
+        "info_url": {
+          "description": "URL of the page with information about the tool",
+          "type": "string"
+        },
+        "install": {
+          "$ref": "#/definitions/installRequirementInfo",
+          "description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation."
+        },
+        "license": {
+          "description": "License name. Use SPDX license identifier if it exists, short name of the license otherwise.",
+          "type": "string"
+        },
+        "version_cmd": {
+          "$ref": "#/definitions/arrayOfStrings",
+          "description": "Command to be executed (along with any extra arguments). The executable be present in one of the export_paths."
+        },
+        "version_regex": {
+          "description": "Regex which is to be applied to version_cmd output to extract the version. By default, the version will be the first capture group of the expression. If version_regex_replace is specified, version will be obtained by doing a substitution using version_regex_replace instead.",
+          "$ref": "#/definitions/regex"
+        },
+        "version_regex_replace": {
+          "description": "If given, this will be used as substitute expression for the regex defined in version_regex, to obtain the version string. Not specifying this is equivalent to setting it to '\\1' (i.e. return the first capture group).",
+          "type": "string"
+        },
+        "strip_container_dirs": {
+          "type": "integer",
+          "description": "If specified, this number of top directory levels will removed when extracting. E.g. if strip_container_dirs=2, archive path a/b/c/d.txt will be extracted as c/d.txt"
+        },
+        "versions": {
+          "type": "array",
+          "description": "List of versions",
+          "items": {
+            "$ref": "#/definitions/versionInfo"
+          }
+        },
+        "platform_overrides": {
+          "type": "array",
+          "description": "List of platform-specific overrides",
+          "items": {
+            "$ref": "#/definitions/platformOverrideInfo"
+          }
+        }
+      },
+      "required": [
+        "description",
+        "export_paths",
+        "version_cmd",
+        "version_regex",
+        "versions",
+        "install",
+        "info_url",
+        "license"
+      ]
+    },
+    "arrayOfStrings": {
+      "description": "Array of strings. Used to represent paths (split into components) and command lines (split into arguments)",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "exportPaths": {
+      "description": "Array of paths to be exported (added to PATH). Each item in the array is relative to the directory where the tool will be installed.",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/arrayOfStrings"
+      }
+    },
+    "envVars": {
+      "description": "Collection of environment variables. Keys and values are the environment variable names and values, respectively.",
+      "type": "object",
+      "patternProperties": {
+        "^([A-Z_0-9]+)+$": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false
+    },
+    "regex": {
+      "description": "A regular expression",
+      "type": "string"
+    },
+    "versionInfo": {
+      "type": "object",
+      "properties": {
+        "name" : {
+          "description": "Version name (used as a directory name)",
+          "type": "string"
+        },
+        "status": {
+            "description": "Determines whether the version is recommended/supported/deprecated",
+            "type": "string",
+            "enum": ["recommended", "supported", "deprecated"]
+        },
+        "linux-i686": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "linux-amd64": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "linux-armel": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "linux-arm64": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "macos": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "win32": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "win64": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "any": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        }
+      }
+    },
+    "platformDownloadInfo": {
+      "description": "Information about download artifact for one platform",
+      "type": "object",
+      "properties": {
+        "sha256": {
+          "type": "string",
+          "description": "SHA256 sum of the file"
+        },
+        "size": {
+          "type": "integer",
+          "description": "Size of the file, in bytes"
+        },
+        "url": {
+          "type": "string",
+          "description": "Download URL"
+        }
+      },
+      "required": [
+        "sha256",
+        "url",
+        "size"
+      ]
+    },
+    "installRequirementInfo": {
+      "description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation.",
+      "type": "string",
+      "enum": ["always", "on_request", "never"]
+    },
+    "platformOverrideInfo": {
+      "description": "Platform-specific values which override the defaults",
+      "type": "object",
+      "properties": {
+        "platforms": {
+          "description": "List of platforms to which this override applies",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "enum": ["linux-i686", "linux-amd64", "linux-armel", "linux-arm64", "macos", "win32", "win64"]
+          }
+        },
+        "export_paths": {
+          "description": "Platform-specific replacement for toolInfo/export_paths",
+          "$ref": "#/definitions/exportPaths"
+        },
+        "export_vars": {
+          "description": "Platform-specific replacement for toolInfo/export_vars",
+          "$ref": "#/definitions/envVars"
+        },
+        "install": {
+          "description": "Platform-specific replacement for toolInfo/install",
+          "$ref": "#/definitions/installRequirementInfo"
+        },
+        "version_cmd": {
+          "description": "Platform-specific replacement for toolInfo/version_cmd",
+          "$ref": "#/definitions/arrayOfStrings"
+        },
+        "version_regex": {
+          "description": "Platform-specific replacement for toolInfo/version_regex",
+          "$ref": "#/definitions/regex"
+        },
+        "version_regex_replace": {
+          "description": "Platform-specific replacement for toolInfo/version_regex_replace",
+          "type": "string"
+        },
+        "strip_container_dirs": {
+          "type": "string",
+          "description": "Platform-specific replacement for toolInfo/strip_container_dirs"
+        }
+      },
+      "required": ["platforms"]
+    }
+  }
+}