Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'feature/idfpy_unknown_targets_fallback' into 'master'

Browse source 
idf.py: run build system target for unknown sub-commands

Closes IDF-748

See merge request espressif/esp-idf!6644
This commit is contained in:
Angus Gratton 2019-11-22 13:22:23 +08:00
commit 7eb89ae868
5 changed files with 159 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
docs/en/api-guides/build-system.rst
View file

@ -67,7 +67,7 @@ The :ref:`getting started guide <get-started-configure>` contains a brief introd
``idf.py`` should be run in an ESP-IDF "project" directory, ie one containing a ``CMakeLists.txt`` file. Older style projects with a Makefile will not work with ``idf.py``.
Type ``idf.py --help`` for a full list of commands. Here are a summary of the most useful ones:
Type ``idf.py --help`` for a list of commands. Here are a summary of the most useful ones:
- ``idf.py menuconfig`` runs the "menuconfig" tool to configure the project.
- ``idf.py build`` will build the project found in the current directory. This can involve multiple steps:
@ -84,6 +84,8 @@ Type ``idf.py --help`` for a full list of commands. Here are a summary of the mo
Multiple ``idf.py`` commands can be combined into one. For example, ``idf.py -p COM4 clean flash monitor`` will clean the source tree, then build the project and flash it to the ESP32 before running the serial monitor.
For commands that are not known to ``idf.py`` an attempt to execute them as a build system target will be made.
.. note:: The environment variables ``ESPPORT`` and ``ESPBAUD`` can be used to set default values for the ``-p`` and ``-b`` options, respectively. Providing these options on the command line overrides the default.
.. _idf.py-size:
@ -101,8 +103,8 @@ The order of multiple ``idf.py`` commands on the same invocation is not importan
idf.py options
^^^^^^^^^^^^^^
To list all available options, run ``idf.py --help``.
To list all available root level options, run ``idf.py --help``. To list options that are specific for a subcommand, run ``idf.py <command> --help``, for example ``idf.py monitor --help``.
Here is a list of some useful options:
- ``-C <dir>`` allows overriding the project directory from the default current working directory.
- ``-B <dir>`` allows overriding the build directory from the default ``build`` subdirectory of the project directory.

45
tools/idf.py
View file

@ -70,7 +70,8 @@ def check_environment():
if "IDF_PATH" in os.environ:
set_idf_path = realpath(os.environ["IDF_PATH"])
if set_idf_path != detected_idf_path:
print("WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. "
print(
"WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. "
"Using the environment variable directory, but results may be unexpected..." %
(set_idf_path, PROG, detected_idf_path))
else:
@ -189,7 +190,8 @@ def init_cli(verbose_output=None):
self.callback(self.name, context, global_args, **action_args)
class Action(click.Command):
def __init__(self,
def __init__(
self,
name=None,
aliases=None,
deprecated=False,
@ -232,12 +234,12 @@ def init_cli(verbose_output=None):
self.help = "\n".join([self.help, aliases_help])
self.short_help = " ".join([aliases_help, self.short_help])
self.unwrapped_callback = self.callback
if self.callback is not None:
callback = self.callback
def wrapped_callback(**action_args):
return Task(
callback=callback,
callback=self.unwrapped_callback,
name=self.name,
dependencies=dependencies,
order_dependencies=order_dependencies,
@ -397,7 +399,8 @@ def init_cli(verbose_output=None):
option = Option(**option_args)
if option.scope.is_shared:
raise FatalError('"%s" is defined for action "%s". '
raise FatalError(
'"%s" is defined for action "%s". '
' "shared" options can be declared only on global level' % (option.name, name))
# Promote options to global if see for the first time
@ -410,8 +413,13 @@ def init_cli(verbose_output=None):
return sorted(filter(lambda name: not self._actions[name].hidden, self._actions))
def get_command(self, ctx, name):
if name in self.commands_with_aliases:
return self._actions.get(self.commands_with_aliases.get(name))
# Trying fallback to build target (from "all" action) if command is not known
else:
return Action(name=name, callback=self._actions.get('fallback').unwrapped_callback)
def _print_closing_message(self, args, actions):
# print a closing message of some kind
#
@ -450,7 +458,8 @@ def init_cli(verbose_output=None):
for o, f in flash_items:
cmd += o + " " + flasher_path(f) + " "
print("%s %s -p %s -b %s --before %s --after %s write_flash %s" % (
print(
"%s %s -p %s -b %s --before %s --after %s write_flash %s" % (
PYTHON,
_safe_relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]),
args.port or "(PORT)",
@ -459,7 +468,8 @@ def init_cli(verbose_output=None):
flasher_args["extra_esptool_args"]["after"],
cmd.strip(),
))
print("or run 'idf.py -p %s %s'" % (
print(
"or run 'idf.py -p %s %s'" % (
args.port or "(PORT)",
key + "-flash" if key != "project" else "flash",
))
@ -483,7 +493,8 @@ def init_cli(verbose_output=None):
[item for item, count in Counter(task.name for task in tasks).items() if count > 1])
if dupplicated_tasks:
dupes = ", ".join('"%s"' % t for t in dupplicated_tasks)
print("WARNING: Command%s found in the list of commands more than once. " %
print(
"WARNING: Command%s found in the list of commands more than once. " %
("s %s are" % dupes if len(dupplicated_tasks) > 1 else " %s is" % dupes) +
"Only first occurence will be executed.")
@ -499,9 +510,9 @@ def init_cli(verbose_output=None):
default = () if option.multiple else option.default
if global_value != default and local_value != default and global_value != local_value:
raise FatalError('Option "%s" provided for "%s" is already defined to a different value. '
"This option can appear at most once in the command line." %
(key, task.name))
raise FatalError(
'Option "%s" provided for "%s" is already defined to a different value. '
"This option can appear at most once in the command line." % (key, task.name))
if local_value != default:
global_args[key] = local_value
@ -537,7 +548,8 @@ def init_cli(verbose_output=None):
# Otherwise invoke it with default set of options
# and put to the front of the list of unprocessed tasks
else:
print('Adding "%s"\'s dependency "%s" to list of commands with default set of options.' %
print(
'Adding "%s"\'s dependency "%s" to list of commands with default set of options.' %
(task.name, dep))
dep_task = ctx.invoke(ctx.command.get_command(ctx, dep))
@ -631,7 +643,11 @@ def init_cli(verbose_output=None):
except NameError:
pass
return CLI(help="ESP-IDF build management", verbose_output=verbose_output, all_actions=all_actions)
cli_help = (
"ESP-IDF CLI build management tool. "
"For commands that are not known to idf.py an attempt to execute it as a build system target will be made.")
return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions)
def main():
@ -709,7 +725,8 @@ if __name__ == "__main__":
# Trying to find best utf-8 locale available on the system and restart python with it
best_locale = _find_usable_locale()
print("Your environment is not configured to handle unicode filenames outside of ASCII range."
print(
"Your environment is not configured to handle unicode filenames outside of ASCII range."
" Environment variable LC_ALL is temporary set to %s for unicode support." % best_locale)
os.environ["LC_ALL"] = best_locale

30
tools/idf_py_actions/constants.py
View file

@ -16,17 +16,23 @@ else:
MAKE_CMD = "make"
MAKE_GENERATOR = "Unix Makefiles"
GENERATORS = [
# ('generator name', 'build command line', 'version command line', 'verbose flag')
("Ninja", ["ninja"], ["ninja", "--version"], "-v"),
(
MAKE_GENERATOR,
[MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)],
[MAKE_CMD, "--version"],
"VERBOSE=1",
),
]
GENERATOR_CMDS = dict((a[0], a[1]) for a in GENERATORS)
GENERATOR_VERBOSE = dict((a[0], a[3]) for a in GENERATORS)
GENERATORS = {
# - command: build command line
# - version: version command line
# - dry_run: command to run in dry run mode
# - verbose_flag: verbose flag
"Ninja": {
"command": ["ninja"],
"version": ["ninja", "--version"],
"dry_run": ["ninja", "-n"],
"verbose_flag": "-v"
},
MAKE_GENERATOR: {
"command": [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)],
"version": [MAKE_CMD, "--version"],
"dry_run": [MAKE_CMD, "-n"],
"verbose_flag": "VERBOSE=1",
}
}
SUPPORTED_TARGETS = ["esp32", "esp32s2beta"]

61
tools/idf_py_actions/core_ext.py
View file

@ -1,16 +1,25 @@
import os
import shutil
import subprocess
import sys
import click
from idf_py_actions.constants import GENERATOR_CMDS, GENERATOR_VERBOSE, SUPPORTED_TARGETS
from idf_py_actions.constants import GENERATORS, SUPPORTED_TARGETS
from idf_py_actions.errors import FatalError
from idf_py_actions.global_options import global_options
from idf_py_actions.tools import ensure_build_directory, idf_version, merge_action_lists, realpath, run_tool
def action_extensions(base_actions, project_path):
def run_target(target_name, args):
generator_cmd = GENERATORS[args.generator]["command"]
if args.verbose:
generator_cmd += [GENERATORS[args.generator]["verbose_flag"]]
run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir)
def build_target(target_name, ctx, args):
"""
Execute the target build system to build target 'target_name'
@ -19,12 +28,23 @@ def action_extensions(base_actions, project_path):
directory (with the specified generator) as needed.
"""
ensure_build_directory(args, ctx.info_name)
generator_cmd = GENERATOR_CMDS[args.generator]
run_target(target_name, args)
if args.verbose:
generator_cmd += [GENERATOR_VERBOSE[args.generator]]
def fallback_target(target_name, ctx, args):
"""
Execute targets that are not explicitly known to idf.py
"""
ensure_build_directory(args, ctx.info_name)
run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir)
try:
subprocess.check_output(GENERATORS[args.generator]["dry_run"] + [target_name], cwd=args.cwd)
except Exception:
raise FatalError(
'command "%s" is not known to idf.py and is not a %s target' %
(target_name, GENERATORS[args.generator].command))
run_target(target_name, args)
def verbose_callback(ctx, param, value):
if not value or ctx.resilient_parsing:
@ -69,7 +89,8 @@ def action_extensions(base_actions, project_path):
return
if not os.path.exists(os.path.join(build_dir, "CMakeCache.txt")):
raise FatalError("Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically "
raise FatalError(
"Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically "
"delete files in this directory. Delete the directory manually to 'clean' it." % build_dir)
red_flags = ["CMakeLists.txt", ".git", ".svn"]
for red in red_flags:
@ -111,7 +132,8 @@ def action_extensions(base_actions, project_path):
def validate_root_options(ctx, args, tasks):
args.project_dir = realpath(args.project_dir)
if args.build_dir is not None and args.project_dir == realpath(args.build_dir):
raise FatalError("Setting the build directory to the project directory is not supported. Suggest dropping "
raise FatalError(
"Setting the build directory to the project directory is not supported. Suggest dropping "
"--build-dir option, the default is a 'build' subdirectory inside the project directory.")
if args.build_dir is None:
args.build_dir = os.path.join(args.project_dir, "build")
@ -165,7 +187,8 @@ def action_extensions(base_actions, project_path):
},
{
"names": ["--ccache/--no-ccache"],
"help": ("Use ccache in build. Disabled by default, unless "
"help": (
"Use ccache in build. Disabled by default, unless "
"IDF_CCACHE_ENABLE environment variable is set to a non-zero value."),
"is_flag": True,
"default": os.getenv("IDF_CCACHE_ENABLE") not in [None, "", "0"],
@ -173,7 +196,7 @@ def action_extensions(base_actions, project_path):
{
"names": ["-G", "--generator"],
"help": "CMake generator.",
"type": click.Choice(GENERATOR_CMDS.keys()),
"type": click.Choice(GENERATORS.keys()),
},
{
"names": ["--dry-run"],
@ -192,7 +215,8 @@ def action_extensions(base_actions, project_path):
"aliases": ["build"],
"callback": build_target,
"short_help": "Build the project.",
"help": ("Build the project. This can involve multiple steps:\n\n"
"help": (
"Build the project. This can involve multiple steps:\n\n"
"1. Create the build directory if needed. "
"The sub-directory 'build' is used to hold build output, "
"although this can be changed with the -B option.\n\n"
@ -282,6 +306,11 @@ def action_extensions(base_actions, project_path):
"help": "Read otadata partition.",
"options": global_options,
},
"fallback": {
"callback": fallback_target,
"help": "Handle for targets not known for idf.py.",
"hidden": True
}
}
}
@ -290,7 +319,8 @@ def action_extensions(base_actions, project_path):
"reconfigure": {
"callback": reconfigure,
"short_help": "Re-run CMake.",
"help": ("Re-run CMake even if it doesn't seem to need re-running. "
"help": (
"Re-run CMake even if it doesn't seem to need re-running. "
"This isn't necessary during normal usage, "
"but can be useful after adding/removing files from the source tree, "
"or when modifying CMake cache variables. "
@ -302,7 +332,8 @@ def action_extensions(base_actions, project_path):
"set-target": {
"callback": set_target,
"short_help": "Set the chip target to build.",
"help": ("Set the chip target to build. This will remove the "
"help": (
"Set the chip target to build. This will remove the "
"existing sdkconfig file and corresponding CMakeCache and "
"create new ones according to the new target.\nFor example, "
"\"idf.py set-target esp32\" will select esp32 as the new chip "
@ -319,7 +350,8 @@ def action_extensions(base_actions, project_path):
"clean": {
"callback": clean,
"short_help": "Delete build output files from the build directory.",
"help": ("Delete build output files from the build directory, "
"help": (
"Delete build output files from the build directory, "
"forcing a 'full rebuild' the next time "
"the project is built. Cleaning doesn't delete "
"CMake configuration output and some other files"),
@ -328,7 +360,8 @@ def action_extensions(base_actions, project_path):
"fullclean": {
"callback": fullclean,
"short_help": "Delete the entire build directory contents.",
"help": ("Delete the entire build directory contents. "
"help": (
"Delete the entire build directory contents. "
"This includes all CMake configuration output."
"The next time the project is built, "
"CMake will configure it from scratch. "

6
tools/idf_py_actions/tools.py
View file

@ -126,9 +126,9 @@ def _detect_cmake_generator(prog_name):
"""
Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found.
"""
for (generator, _, version_check, _) in GENERATORS:
if executable_exists(version_check):
return generator
for (generator_name, generator) in GENERATORS.items():
if executable_exists(generator["version"]):
return generator_name
raise FatalError("To use %s, either the 'ninja' or 'GNU make' build tool must be available in the PATH" % prog_name)