tools: add metadata file and idf_tools.py

This commit is contained in:
Ivan Grokhotkov 2019-04-17 15:30:30 +08:00
parent 8d1a9c07a0
commit 8f7e01baed
6 changed files with 1964 additions and 0 deletions

View file

@ -601,6 +601,14 @@ test_idf_size:
- cd ${IDF_PATH}/tools/test_idf_size
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
test_idf_tools:
<<: *host_test_template
script:
# Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment
- export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/opt/espressif${CUSTOM_TOOLCHAIN_PATH:+\|${CUSTOM_TOOLCHAIN_PATH}}" | tr "\n" ":"); echo ${p%:})
- cd ${IDF_PATH}/tools/test_idf_tools
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
test_esp_err_to_name_on_host:
<<: *host_test_template
artifacts:

View file

@ -53,6 +53,7 @@ tools/gen_esp_err_to_name.py
tools/idf.py
tools/idf_monitor.py
tools/idf_size.py
tools/idf_tools.py
tools/kconfig/check.sh
tools/kconfig/lxdialog/check-lxdialog.sh
tools/kconfig/merge_config.sh
@ -67,6 +68,7 @@ tools/mass_mfg/mfg_gen.py
tools/test_check_kconfigs.py
tools/test_idf_monitor/run_test_idf_monitor.py
tools/test_idf_size/test.sh
tools/test_idf_tools/test_idf_tools.py
tools/unit-test-app/unit_test.py
tools/windows/eclipse_make.sh
tools/windows/tool_setup/build_installer.sh

1243
tools/idf_tools.py Executable file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,130 @@
#!/usr/bin/env python
#
# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import unittest
import tempfile
import shutil
try:
from contextlib import redirect_stdout
except ImportError:
import contextlib
@contextlib.contextmanager
def redirect_stdout(target):
original = sys.stdout
sys.stdout = target
yield
sys.stdout = original
try:
from cStringIO import StringIO
except ImportError:
from io import StringIO
# Need to do this before importing idf_tools.py
os.environ['IDF_MAINTAINER'] = '1'
try:
import idf_tools
except ImportError:
sys.path.append('..')
import idf_tools
class TestUsage(unittest.TestCase):
def test_usage_basic(self):
old_tools_dir = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT)
mirror_prefix_map = None
if os.path.exists(old_tools_dir):
mirror_prefix_map = 'https://dl.espressif.com/dl,file://' + os.path.join(old_tools_dir, 'dist')
mirror_prefix_map += ';https://github.com/espressif/.*/releases/download/.*/,file://' + os.path.join(old_tools_dir, 'dist', '')
if mirror_prefix_map:
print('Using IDF_MIRROR_PREFIX_MAP={}'.format(mirror_prefix_map))
os.environ['IDF_MIRROR_PREFIX_MAP'] = mirror_prefix_map
temp_tools_dir = tempfile.mkdtemp(prefix='idf_tools_tmp')
print('Using IDF_TOOLS_PATH={}'.format(temp_tools_dir))
os.environ['IDF_TOOLS_PATH'] = temp_tools_dir
self.addCleanup(shutil.rmtree, temp_tools_dir)
output_stream = StringIO()
with redirect_stdout(output_stream):
idf_tools.main(['list'])
output = output_stream.getvalue()
xtensa_esp32_elf_version = '1.22.0-80-g6c4433a-5.2.0'
esp32ulp_version = '2.28.51.20170517'
self.assertIn('* xtensa-esp32-elf:', output)
self.assertIn('- %s (recommended)' % xtensa_esp32_elf_version, output)
self.assertIn('* esp32ulp-elf', output)
self.assertIn('- %s (recommended)' % esp32ulp_version, output)
output_stream = StringIO()
with redirect_stdout(output_stream):
idf_tools.main(['install'])
output = output_stream.getvalue()
self.assertIn('Installing esp32ulp-elf@' + esp32ulp_version, output)
self.assertIn('Downloading binutils-esp32ulp', output)
self.assertIn('Installing xtensa-esp32-elf@' + xtensa_esp32_elf_version, output)
self.assertIn('Downloading xtensa-esp32-elf', output)
self.assertIn('to ' + os.path.join(temp_tools_dir, 'dist'), output)
output_stream = StringIO()
with redirect_stdout(output_stream):
idf_tools.main(['check'])
output = output_stream.getvalue()
self.assertIn('version installed in tools directory: ' + esp32ulp_version, output)
self.assertIn('version installed in tools directory: ' + xtensa_esp32_elf_version, output)
output_stream = StringIO()
with redirect_stdout(output_stream):
idf_tools.main(['export'])
output = output_stream.getvalue()
self.assertIn('%s/tools/esp32ulp-elf/%s/esp32ulp-elf-binutils/bin' %
(temp_tools_dir, esp32ulp_version), output)
self.assertIn('%s/tools/xtensa-esp32-elf/%s/xtensa-esp32-elf/bin' %
(temp_tools_dir, xtensa_esp32_elf_version), output)
class TestMaintainer(unittest.TestCase):
def test_validation(self):
idf_tools.main(['validate'])
def test_json_rewrite(self):
idf_tools.main(['rewrite'])
idf_path = os.getenv('IDF_PATH')
if not idf_path:
self.fail('IDF_PATH needs to be set to run this test')
with open(os.path.join(idf_path, 'tools/tools.json'), 'r') as f:
json_old = f.read()
with open(os.path.join(idf_path, 'tools/tools.new.json'), 'r') as f:
json_new = f.read()
self.assertEqual(json_old, json_new)
if __name__ == '__main__':
unittest.main()

347
tools/tools.json Normal file
View file

@ -0,0 +1,347 @@
{
"tools": [
{
"description": "Toolchain for Xtensa (ESP32) based on GCC",
"export_paths": [
[
"xtensa-esp32-elf",
"bin"
]
],
"export_vars": {},
"info_url": "https://github.com/espressif/crosstool-NG",
"install": "always",
"license": "GPL-3.0-with-GCC-exception",
"name": "xtensa-esp32-elf",
"version_cmd": [
"xtensa-esp32-elf-gcc",
"--version"
],
"version_regex": "\\(crosstool-NG\\s+(?:crosstool-ng-)?([0-9a-z\\.\\-]+)\\)\\s*([0-9\\.]+)",
"version_regex_replace": "\\1-\\2",
"versions": [
{
"name": "1.22.0-80-g6c4433a5-5.2.0",
"status": "recommended",
"win32": {
"sha256": "f217fccbeaaa8c92db239036e0d6202458de4488b954a3a38f35ac2ec48058a4",
"size": 125719261,
"url": "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
},
"win64": {
"sha256": "f217fccbeaaa8c92db239036e0d6202458de4488b954a3a38f35ac2ec48058a4",
"size": 125719261,
"url": "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
}
},
{
"linux-amd64": {
"sha256": "3fe96c151d46c1d4e5edc6ed690851b8e53634041114bad04729bc16b0445156",
"size": 44219107,
"url": "https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22.0-80-g6c4433a-5.2.0.tar.gz"
},
"linux-i686": {
"sha256": "b4055695ffc2dfc0bcb6dafdc2572a6e01151c4179ef5fa972b3fcb2183eb155",
"size": 45566336,
"url": "https://dl.espressif.com/dl/xtensa-esp32-elf-linux32-1.22.0-80-g6c4433a-5.2.0.tar.gz"
},
"macos": {
"sha256": "a4307a97945d2f2f2745f415fbe80d727750e19f91f9a1e7e2f8a6065652f9da",
"size": 46517409,
"url": "https://dl.espressif.com/dl/xtensa-esp32-elf-osx-1.22.0-80-g6c4433a-5.2.0.tar.gz"
},
"name": "1.22.0-80-g6c4433a-5.2.0",
"status": "recommended"
}
]
},
{
"description": "Toolchain for ESP32 ULP coprocessor",
"export_paths": [
[
"esp32ulp-elf-binutils",
"bin"
]
],
"export_vars": {},
"info_url": "https://github.com/espressif/binutils-esp32ulp",
"install": "always",
"license": "GPL-2.0-or-later",
"name": "esp32ulp-elf",
"version_cmd": [
"esp32ulp-elf-as",
"--version"
],
"version_regex": "\\(GNU Binutils\\)\\s+([0-9a-z\\.\\-]+)",
"versions": [
{
"linux-amd64": {
"sha256": "c1bbcd65e1e30c7312a50344c8dbc70c2941580a79aa8f8abbce8e0e90c79566",
"size": 8246604,
"url": "https://dl.espressif.com/dl/binutils-esp32ulp-linux64-2.28.51-esp32ulp-20180809.tar.gz"
},
"macos": {
"sha256": "c92937d85cc9a90eb6c6099ce767ca021108c18c94e34bd7b1fa0cde168f94a0",
"size": 5726662,
"url": "https://dl.espressif.com/dl/binutils-esp32ulp-macos-2.28.51-esp32ulp-20180809.tar.gz"
},
"name": "2.28.51.20170517",
"status": "recommended",
"win32": {
"sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd",
"size": 12231559,
"url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip"
},
"win64": {
"sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd",
"size": 12231559,
"url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip"
}
}
]
},
{
"description": "CMake build system",
"export_paths": [
[
"bin"
]
],
"export_vars": {},
"info_url": "https://github.com/Kitware/CMake",
"install": "on_request",
"license": "BSD-3-Clause",
"name": "cmake",
"platform_overrides": [
{
"install": "always",
"platforms": [
"win32",
"win64"
]
},
{
"export_paths": [
[
"CMake.app",
"Contents",
"bin"
]
],
"platforms": [
"macos"
]
}
],
"strip_container_dirs": 1,
"version_cmd": [
"cmake",
"--version"
],
"version_regex": "cmake version ([0-9.]+)",
"versions": [
{
"linux-amd64": {
"sha256": "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76",
"size": 38405896,
"url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz"
},
"macos": {
"sha256": "fef537614d73fda848f6168273b6c7ba45f850484533361e7bc50ac1d315f780",
"size": 32062124,
"url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Darwin-x86_64.tar.gz"
},
"name": "3.13.4",
"status": "recommended",
"win32": {
"sha256": "28daf772f55d817a13ef14e25af2a5569f8326dac66a6aa3cc5208cf1f8e943f",
"size": 26385104,
"url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win32-x86.zip"
},
"win64": {
"sha256": "bcd477d49e4a9400b41213d53450b474beaedb264631693c958ef9affa8e5623",
"size": 29696565,
"url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win64-x64.zip"
}
}
]
},
{
"description": "OpenOCD for ESP32",
"export_paths": [
[
"openocd-esp32",
"bin"
]
],
"export_vars": {
"OPENOCD_SCRIPTS": "${TOOL_PATH}/openocd-esp32/share/openocd/scripts"
},
"info_url": "https://github.com/espressif/openocd-esp32",
"install": "always",
"license": "GPL-2.0-only",
"name": "openocd-esp32",
"version_cmd": [
"openocd",
"--version"
],
"version_regex": "Open On-Chip Debugger\\s+([a-z0-9.-]+)\\s+",
"versions": [
{
"linux-amd64": {
"sha256": "e5b5579edffde090e426b4995b346e281843bf84394f8e68c8e41bd1e4c576bd",
"size": 1681596,
"url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-linux64-0.10.0-esp32-20190313.tar.gz"
},
"macos": {
"sha256": "09504eea5aa92646a117f16573c95b34e04b4010791a2f8fefcd2bd8c430f081",
"size": 1760536,
"url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-macos-0.10.0-esp32-20190313.tar.gz"
},
"name": "v0.10.0-esp32-20190313",
"status": "recommended",
"win32": {
"sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2",
"size": 2098513,
"url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip"
},
"win64": {
"sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2",
"size": 2098513,
"url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip"
}
}
]
},
{
"description": "menuconfig tool",
"export_paths": [
[
""
]
],
"export_vars": {},
"info_url": "https://github.com/espressif/kconfig-frontends",
"install": "never",
"license": "GPL-2.0-only",
"name": "mconf",
"platform_overrides": [
{
"install": "always",
"platforms": [
"win32",
"win64"
]
}
],
"strip_container_dirs": 1,
"version_cmd": [
"mconf-idf",
"-v"
],
"version_regex": "mconf-idf version mconf-([a-z0-9.-]+)-win32",
"versions": [
{
"name": "v4.6.0.0-idf-20190313",
"status": "recommended",
"win32": {
"sha256": "051bef09c782bc31b737b047808d1b6588b0965101b77dc979af9139773c4b4f",
"size": 826167,
"url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190313/mconf-v4.6.0.0-idf-20190313-win32.zip"
},
"win64": {
"sha256": "051bef09c782bc31b737b047808d1b6588b0965101b77dc979af9139773c4b4f",
"size": 826167,
"url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190313/mconf-v4.6.0.0-idf-20190313-win32.zip"
}
}
]
},
{
"description": "Ninja build system",
"export_paths": [
[
""
]
],
"export_vars": {},
"info_url": "https://github.com/ninja-build/ninja",
"install": "on_request",
"license": "Apache-2.0",
"name": "ninja",
"platform_overrides": [
{
"install": "always",
"platforms": [
"win32",
"win64"
]
}
],
"version_cmd": [
"ninja",
"--version"
],
"version_regex": "([0-9.]+)",
"versions": [
{
"linux-amd64": {
"sha256": "978fd9e26c2db8d33392c6daef50e9edac0a3db6680710a9f9ad47e01f3e49b7",
"size": 85276,
"url": "https://dl.espressif.com/dl/ninja-1.9.0-linux64.tar.gz"
},
"macos": {
"sha256": "9504cd1783ef3c242d06330a50d54dc8f838b605f5fc3e892c47254929f7350c",
"size": 91457,
"url": "https://dl.espressif.com/dl/ninja-1.9.0-osx.tar.gz"
},
"name": "1.9.0",
"status": "recommended",
"win64": {
"sha256": "2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9",
"size": 254497,
"url": "https://dl.espressif.com/dl/ninja-1.9.0-win64.zip"
}
}
]
},
{
"description": "Ccache (compiler cache)",
"export_paths": [
[
""
]
],
"export_vars": {},
"info_url": "https://github.com/ccache/ccache",
"install": "never",
"license": "GPL-3.0-or-later",
"name": "ccache",
"platform_overrides": [
{
"install": "always",
"platforms": [
"win64"
]
}
],
"version_cmd": [
"ccache.exe",
"--version"
],
"version_regex": "ccache version ([0-9.]+)",
"versions": [
{
"name": "3.7",
"status": "recommended",
"win64": {
"sha256": "37e833f3f354f1145503533e776c1bd44ec2e77ff8a2476a1d2039b0b10c78d6",
"size": 142401,
"url": "https://dl.espressif.com/dl/ccache-3.7-w64.zip"
}
}
]
}
],
"version": 1
}

234
tools/tools_schema.json Normal file
View file

@ -0,0 +1,234 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/espressif/esp-idf/blob/master/tools/tools-schema.json",
"type": "object",
"properties": {
"version": {
"type": "integer",
"description": "Metadata file version"
},
"tools": {
"type": "array",
"description": "List of tools",
"items": {
"$ref": "#/definitions/toolInfo"
}
}
},
"required": [
"version",
"tools"
],
"definitions": {
"toolInfo": {
"type": "object",
"description": "Information about one tool",
"properties": {
"name" : {
"description": "Tool name (used as a directory name)",
"type": "string"
},
"description" : {
"description": "A short (one sentence) description of the tool.",
"type": "string"
},
"export_paths": {
"$ref": "#/definitions/exportPaths"
},
"export_vars": {
"$ref": "#/definitions/envVars",
"description": "Some variable expansions are done on the values. 1) ${TOOL_PATH} is replaced with the directory where the tool is installed."
},
"info_url": {
"description": "URL of the page with information about the tool",
"type": "string"
},
"install": {
"$ref": "#/definitions/installRequirementInfo",
"description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation."
},
"license": {
"description": "License name. Use SPDX license identifier if it exists, short name of the license otherwise.",
"type": "string"
},
"version_cmd": {
"$ref": "#/definitions/arrayOfStrings",
"description": "Command to be executed (along with any extra arguments). The executable be present in one of the export_paths."
},
"version_regex": {
"description": "Regex which is to be applied to version_cmd output to extract the version. By default, the version will be the first capture group of the expression. If version_regex_replace is specified, version will be obtained by doing a substitution using version_regex_replace instead.",
"$ref": "#/definitions/regex"
},
"version_regex_replace": {
"description": "If given, this will be used as substitute expression for the regex defined in version_regex, to obtain the version string. Not specifying this is equivalent to setting it to '\\1' (i.e. return the first capture group).",
"type": "string"
},
"strip_container_dirs": {
"type": "integer",
"description": "If specified, this number of top directory levels will removed when extracting. E.g. if strip_container_dirs=2, archive path a/b/c/d.txt will be extracted as c/d.txt"
},
"versions": {
"type": "array",
"description": "List of versions",
"items": {
"$ref": "#/definitions/versionInfo"
}
},
"platform_overrides": {
"type": "array",
"description": "List of platform-specific overrides",
"items": {
"$ref": "#/definitions/platformOverrideInfo"
}
}
},
"required": [
"description",
"export_paths",
"version_cmd",
"version_regex",
"versions",
"install",
"info_url",
"license"
]
},
"arrayOfStrings": {
"description": "Array of strings. Used to represent paths (split into components) and command lines (split into arguments)",
"type": "array",
"items": {
"type": "string"
}
},
"exportPaths": {
"description": "Array of paths to be exported (added to PATH). Each item in the array is relative to the directory where the tool will be installed.",
"type": "array",
"items": {
"$ref": "#/definitions/arrayOfStrings"
}
},
"envVars": {
"description": "Collection of environment variables. Keys and values are the environment variable names and values, respectively.",
"type": "object",
"patternProperties": {
"^([A-Z_0-9]+)+$": {
"type": "string"
}
},
"additionalProperties": false
},
"regex": {
"description": "A regular expression",
"type": "string"
},
"versionInfo": {
"type": "object",
"properties": {
"name" : {
"description": "Version name (used as a directory name)",
"type": "string"
},
"status": {
"description": "Determines whether the version is recommended/supported/deprecated",
"type": "string",
"enum": ["recommended", "supported", "deprecated"]
},
"linux-i686": {
"$ref": "#/definitions/platformDownloadInfo"
},
"linux-amd64": {
"$ref": "#/definitions/platformDownloadInfo"
},
"linux-armel": {
"$ref": "#/definitions/platformDownloadInfo"
},
"linux-arm64": {
"$ref": "#/definitions/platformDownloadInfo"
},
"macos": {
"$ref": "#/definitions/platformDownloadInfo"
},
"win32": {
"$ref": "#/definitions/platformDownloadInfo"
},
"win64": {
"$ref": "#/definitions/platformDownloadInfo"
},
"any": {
"$ref": "#/definitions/platformDownloadInfo"
}
}
},
"platformDownloadInfo": {
"description": "Information about download artifact for one platform",
"type": "object",
"properties": {
"sha256": {
"type": "string",
"description": "SHA256 sum of the file"
},
"size": {
"type": "integer",
"description": "Size of the file, in bytes"
},
"url": {
"type": "string",
"description": "Download URL"
}
},
"required": [
"sha256",
"url",
"size"
]
},
"installRequirementInfo": {
"description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation.",
"type": "string",
"enum": ["always", "on_request", "never"]
},
"platformOverrideInfo": {
"description": "Platform-specific values which override the defaults",
"type": "object",
"properties": {
"platforms": {
"description": "List of platforms to which this override applies",
"type": "array",
"items": {
"type": "string",
"enum": ["linux-i686", "linux-amd64", "linux-armel", "linux-arm64", "macos", "win32", "win64"]
}
},
"export_paths": {
"description": "Platform-specific replacement for toolInfo/export_paths",
"$ref": "#/definitions/exportPaths"
},
"export_vars": {
"description": "Platform-specific replacement for toolInfo/export_vars",
"$ref": "#/definitions/envVars"
},
"install": {
"description": "Platform-specific replacement for toolInfo/install",
"$ref": "#/definitions/installRequirementInfo"
},
"version_cmd": {
"description": "Platform-specific replacement for toolInfo/version_cmd",
"$ref": "#/definitions/arrayOfStrings"
},
"version_regex": {
"description": "Platform-specific replacement for toolInfo/version_regex",
"$ref": "#/definitions/regex"
},
"version_regex_replace": {
"description": "Platform-specific replacement for toolInfo/version_regex_replace",
"type": "string"
},
"strip_container_dirs": {
"type": "string",
"description": "Platform-specific replacement for toolInfo/strip_container_dirs"
}
},
"required": ["platforms"]
}
}
}