From 18c594e2508a44b89840e53d92c6e0ac06d9d2de Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Fri, 8 Nov 2019 16:46:02 +0100 Subject: [PATCH 1/2] idf.py: run build system target for unknown commands --- tools/idf.py | 97 ++++++++++++++---------- tools/idf_py_actions/constants.py | 30 +++++--- tools/idf_py_actions/core_ext.py | 119 +++++++++++++++++++----------- tools/idf_py_actions/tools.py | 6 +- 4 files changed, 154 insertions(+), 98 deletions(-) diff --git a/tools/idf.py b/tools/idf.py index 129b3be4c..b4077a194 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -70,9 +70,10 @@ def check_environment(): if "IDF_PATH" in os.environ: set_idf_path = realpath(os.environ["IDF_PATH"]) if set_idf_path != detected_idf_path: - print("WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. " - "Using the environment variable directory, but results may be unexpected..." % - (set_idf_path, PROG, detected_idf_path)) + print( + "WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. " + "Using the environment variable directory, but results may be unexpected..." % + (set_idf_path, PROG, detected_idf_path)) else: print("Setting IDF_PATH environment variable: %s" % detected_idf_path) os.environ["IDF_PATH"] = detected_idf_path @@ -189,14 +190,15 @@ def init_cli(verbose_output=None): self.callback(self.name, context, global_args, **action_args) class Action(click.Command): - def __init__(self, - name=None, - aliases=None, - deprecated=False, - dependencies=None, - order_dependencies=None, - hidden=False, - **kwargs): + def __init__( + self, + name=None, + aliases=None, + deprecated=False, + dependencies=None, + order_dependencies=None, + hidden=False, + **kwargs): super(Action, self).__init__(name, **kwargs) self.name = self.name or self.callback.__name__ @@ -232,12 +234,12 @@ def init_cli(verbose_output=None): self.help = "\n".join([self.help, aliases_help]) self.short_help = " ".join([aliases_help, self.short_help]) + self.unwrapped_callback = self.callback if self.callback is not None: - callback = self.callback def wrapped_callback(**action_args): return Task( - callback=callback, + callback=self.unwrapped_callback, name=self.name, dependencies=dependencies, order_dependencies=order_dependencies, @@ -397,8 +399,9 @@ def init_cli(verbose_output=None): option = Option(**option_args) if option.scope.is_shared: - raise FatalError('"%s" is defined for action "%s". ' - ' "shared" options can be declared only on global level' % (option.name, name)) + raise FatalError( + '"%s" is defined for action "%s". ' + ' "shared" options can be declared only on global level' % (option.name, name)) # Promote options to global if see for the first time if option.scope.is_global and option.name not in [o.name for o in self.params]: @@ -410,7 +413,12 @@ def init_cli(verbose_output=None): return sorted(filter(lambda name: not self._actions[name].hidden, self._actions)) def get_command(self, ctx, name): - return self._actions.get(self.commands_with_aliases.get(name)) + if name in self.commands_with_aliases: + return self._actions.get(self.commands_with_aliases.get(name)) + + # Trying fallback to build target (from "all" action) if command is not known + else: + return Action(name=name, callback=self._actions.get('fallback').unwrapped_callback) def _print_closing_message(self, args, actions): # print a closing message of some kind @@ -450,19 +458,21 @@ def init_cli(verbose_output=None): for o, f in flash_items: cmd += o + " " + flasher_path(f) + " " - print("%s %s -p %s -b %s --before %s --after %s write_flash %s" % ( - PYTHON, - _safe_relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]), - args.port or "(PORT)", - args.baud, - flasher_args["extra_esptool_args"]["before"], - flasher_args["extra_esptool_args"]["after"], - cmd.strip(), - )) - print("or run 'idf.py -p %s %s'" % ( - args.port or "(PORT)", - key + "-flash" if key != "project" else "flash", - )) + print( + "%s %s -p %s -b %s --before %s --after %s write_flash %s" % ( + PYTHON, + _safe_relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]), + args.port or "(PORT)", + args.baud, + flasher_args["extra_esptool_args"]["before"], + flasher_args["extra_esptool_args"]["after"], + cmd.strip(), + )) + print( + "or run 'idf.py -p %s %s'" % ( + args.port or "(PORT)", + key + "-flash" if key != "project" else "flash", + )) if "all" in actions or "build" in actions: print_flashing_message("Project", "project") @@ -483,9 +493,10 @@ def init_cli(verbose_output=None): [item for item, count in Counter(task.name for task in tasks).items() if count > 1]) if dupplicated_tasks: dupes = ", ".join('"%s"' % t for t in dupplicated_tasks) - print("WARNING: Command%s found in the list of commands more than once. " % - ("s %s are" % dupes if len(dupplicated_tasks) > 1 else " %s is" % dupes) + - "Only first occurence will be executed.") + print( + "WARNING: Command%s found in the list of commands more than once. " % + ("s %s are" % dupes if len(dupplicated_tasks) > 1 else " %s is" % dupes) + + "Only first occurence will be executed.") # Set propagated global options. # These options may be set on one subcommand, but available in the list of global arguments @@ -499,9 +510,9 @@ def init_cli(verbose_output=None): default = () if option.multiple else option.default if global_value != default and local_value != default and global_value != local_value: - raise FatalError('Option "%s" provided for "%s" is already defined to a different value. ' - "This option can appear at most once in the command line." % - (key, task.name)) + raise FatalError( + 'Option "%s" provided for "%s" is already defined to a different value. ' + "This option can appear at most once in the command line." % (key, task.name)) if local_value != default: global_args[key] = local_value @@ -537,8 +548,9 @@ def init_cli(verbose_output=None): # Otherwise invoke it with default set of options # and put to the front of the list of unprocessed tasks else: - print('Adding "%s"\'s dependency "%s" to list of commands with default set of options.' % - (task.name, dep)) + print( + 'Adding "%s"\'s dependency "%s" to list of commands with default set of options.' % + (task.name, dep)) dep_task = ctx.invoke(ctx.command.get_command(ctx, dep)) # Remove options with global scope from invoke tasks because they are alread in global_args @@ -631,7 +643,11 @@ def init_cli(verbose_output=None): except NameError: pass - return CLI(help="ESP-IDF build management", verbose_output=verbose_output, all_actions=all_actions) + cli_help = ( + "ESP-IDF CLI build management tool. " + "For commands that are not known to idf.py an attempt to execute it as a build system target will be made.") + + return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions) def main(): @@ -709,8 +725,9 @@ if __name__ == "__main__": # Trying to find best utf-8 locale available on the system and restart python with it best_locale = _find_usable_locale() - print("Your environment is not configured to handle unicode filenames outside of ASCII range." - " Environment variable LC_ALL is temporary set to %s for unicode support." % best_locale) + print( + "Your environment is not configured to handle unicode filenames outside of ASCII range." + " Environment variable LC_ALL is temporary set to %s for unicode support." % best_locale) os.environ["LC_ALL"] = best_locale ret = subprocess.call([sys.executable] + sys.argv, env=os.environ) diff --git a/tools/idf_py_actions/constants.py b/tools/idf_py_actions/constants.py index 4cfc6a02c..a7e81a2b2 100644 --- a/tools/idf_py_actions/constants.py +++ b/tools/idf_py_actions/constants.py @@ -16,17 +16,23 @@ else: MAKE_CMD = "make" MAKE_GENERATOR = "Unix Makefiles" -GENERATORS = [ - # ('generator name', 'build command line', 'version command line', 'verbose flag') - ("Ninja", ["ninja"], ["ninja", "--version"], "-v"), - ( - MAKE_GENERATOR, - [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)], - [MAKE_CMD, "--version"], - "VERBOSE=1", - ), -] -GENERATOR_CMDS = dict((a[0], a[1]) for a in GENERATORS) -GENERATOR_VERBOSE = dict((a[0], a[3]) for a in GENERATORS) +GENERATORS = { + # - command: build command line + # - version: version command line + # - dry_run: command to run in dry run mode + # - verbose_flag: verbose flag + "Ninja": { + "command": ["ninja"], + "version": ["ninja", "--version"], + "dry_run": ["ninja", "-n"], + "verbose_flag": "-v" + }, + MAKE_GENERATOR: { + "command": [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)], + "version": [MAKE_CMD, "--version"], + "dry_run": [MAKE_CMD, "-n"], + "verbose_flag": "VERBOSE=1", + } +} SUPPORTED_TARGETS = ["esp32", "esp32s2beta"] diff --git a/tools/idf_py_actions/core_ext.py b/tools/idf_py_actions/core_ext.py index 7f72a2de7..7eff74e5f 100644 --- a/tools/idf_py_actions/core_ext.py +++ b/tools/idf_py_actions/core_ext.py @@ -1,16 +1,25 @@ import os import shutil +import subprocess import sys import click -from idf_py_actions.constants import GENERATOR_CMDS, GENERATOR_VERBOSE, SUPPORTED_TARGETS +from idf_py_actions.constants import GENERATORS, SUPPORTED_TARGETS from idf_py_actions.errors import FatalError from idf_py_actions.global_options import global_options from idf_py_actions.tools import ensure_build_directory, idf_version, merge_action_lists, realpath, run_tool def action_extensions(base_actions, project_path): + def run_target(target_name, args): + generator_cmd = GENERATORS[args.generator]["command"] + + if args.verbose: + generator_cmd += [GENERATORS[args.generator]["verbose_flag"]] + + run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir) + def build_target(target_name, ctx, args): """ Execute the target build system to build target 'target_name' @@ -19,12 +28,23 @@ def action_extensions(base_actions, project_path): directory (with the specified generator) as needed. """ ensure_build_directory(args, ctx.info_name) - generator_cmd = GENERATOR_CMDS[args.generator] + run_target(target_name, args) - if args.verbose: - generator_cmd += [GENERATOR_VERBOSE[args.generator]] + def fallback_target(target_name, ctx, args): + """ + Execute targets that are not explicitly known to idf.py + """ + ensure_build_directory(args, ctx.info_name) - run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir) + try: + subprocess.check_output(GENERATORS[args.generator]["dry_run"] + [target_name], cwd=args.cwd) + + except Exception: + raise FatalError( + 'command "%s" is not known to idf.py and is not a %s target' % + (target_name, GENERATORS[args.generator].command)) + + run_target(target_name, args) def verbose_callback(ctx, param, value): if not value or ctx.resilient_parsing: @@ -69,8 +89,9 @@ def action_extensions(base_actions, project_path): return if not os.path.exists(os.path.join(build_dir, "CMakeCache.txt")): - raise FatalError("Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically " - "delete files in this directory. Delete the directory manually to 'clean' it." % build_dir) + raise FatalError( + "Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically " + "delete files in this directory. Delete the directory manually to 'clean' it." % build_dir) red_flags = ["CMakeLists.txt", ".git", ".svn"] for red in red_flags: red = os.path.join(build_dir, red) @@ -111,8 +132,9 @@ def action_extensions(base_actions, project_path): def validate_root_options(ctx, args, tasks): args.project_dir = realpath(args.project_dir) if args.build_dir is not None and args.project_dir == realpath(args.build_dir): - raise FatalError("Setting the build directory to the project directory is not supported. Suggest dropping " - "--build-dir option, the default is a 'build' subdirectory inside the project directory.") + raise FatalError( + "Setting the build directory to the project directory is not supported. Suggest dropping " + "--build-dir option, the default is a 'build' subdirectory inside the project directory.") if args.build_dir is None: args.build_dir = os.path.join(args.project_dir, "build") args.build_dir = realpath(args.build_dir) @@ -165,15 +187,16 @@ def action_extensions(base_actions, project_path): }, { "names": ["--ccache/--no-ccache"], - "help": ("Use ccache in build. Disabled by default, unless " - "IDF_CCACHE_ENABLE environment variable is set to a non-zero value."), + "help": ( + "Use ccache in build. Disabled by default, unless " + "IDF_CCACHE_ENABLE environment variable is set to a non-zero value."), "is_flag": True, "default": os.getenv("IDF_CCACHE_ENABLE") not in [None, "", "0"], }, { "names": ["-G", "--generator"], "help": "CMake generator.", - "type": click.Choice(GENERATOR_CMDS.keys()), + "type": click.Choice(GENERATORS.keys()), }, { "names": ["--dry-run"], @@ -192,15 +215,16 @@ def action_extensions(base_actions, project_path): "aliases": ["build"], "callback": build_target, "short_help": "Build the project.", - "help": ("Build the project. This can involve multiple steps:\n\n" - "1. Create the build directory if needed. " - "The sub-directory 'build' is used to hold build output, " - "although this can be changed with the -B option.\n\n" - "2. Run CMake as necessary to configure the project " - "and generate build files for the main build tool.\n\n" - "3. Run the main build tool (Ninja or GNU Make). " - "By default, the build tool is automatically detected " - "but it can be explicitly set by passing the -G option to idf.py.\n\n"), + "help": ( + "Build the project. This can involve multiple steps:\n\n" + "1. Create the build directory if needed. " + "The sub-directory 'build' is used to hold build output, " + "although this can be changed with the -B option.\n\n" + "2. Run CMake as necessary to configure the project " + "and generate build files for the main build tool.\n\n" + "3. Run the main build tool (Ninja or GNU Make). " + "By default, the build tool is automatically detected " + "but it can be explicitly set by passing the -G option to idf.py.\n\n"), "options": global_options, "order_dependencies": [ "reconfigure", @@ -282,6 +306,11 @@ def action_extensions(base_actions, project_path): "help": "Read otadata partition.", "options": global_options, }, + "fallback": { + "callback": fallback_target, + "help": "Handle for targets not known for idf.py.", + "hidden": True + } } } @@ -290,23 +319,25 @@ def action_extensions(base_actions, project_path): "reconfigure": { "callback": reconfigure, "short_help": "Re-run CMake.", - "help": ("Re-run CMake even if it doesn't seem to need re-running. " - "This isn't necessary during normal usage, " - "but can be useful after adding/removing files from the source tree, " - "or when modifying CMake cache variables. " - "For example, \"idf.py -DNAME='VALUE' reconfigure\" " - 'can be used to set variable "NAME" in CMake cache to value "VALUE".'), + "help": ( + "Re-run CMake even if it doesn't seem to need re-running. " + "This isn't necessary during normal usage, " + "but can be useful after adding/removing files from the source tree, " + "or when modifying CMake cache variables. " + "For example, \"idf.py -DNAME='VALUE' reconfigure\" " + 'can be used to set variable "NAME" in CMake cache to value "VALUE".'), "options": global_options, "order_dependencies": ["menuconfig", "fullclean"], }, "set-target": { "callback": set_target, "short_help": "Set the chip target to build.", - "help": ("Set the chip target to build. This will remove the " - "existing sdkconfig file and corresponding CMakeCache and " - "create new ones according to the new target.\nFor example, " - "\"idf.py set-target esp32\" will select esp32 as the new chip " - "target."), + "help": ( + "Set the chip target to build. This will remove the " + "existing sdkconfig file and corresponding CMakeCache and " + "create new ones according to the new target.\nFor example, " + "\"idf.py set-target esp32\" will select esp32 as the new chip " + "target."), "arguments": [ { "names": ["idf-target"], @@ -319,22 +350,24 @@ def action_extensions(base_actions, project_path): "clean": { "callback": clean, "short_help": "Delete build output files from the build directory.", - "help": ("Delete build output files from the build directory, " - "forcing a 'full rebuild' the next time " - "the project is built. Cleaning doesn't delete " - "CMake configuration output and some other files"), + "help": ( + "Delete build output files from the build directory, " + "forcing a 'full rebuild' the next time " + "the project is built. Cleaning doesn't delete " + "CMake configuration output and some other files"), "order_dependencies": ["fullclean"], }, "fullclean": { "callback": fullclean, "short_help": "Delete the entire build directory contents.", - "help": ("Delete the entire build directory contents. " - "This includes all CMake configuration output." - "The next time the project is built, " - "CMake will configure it from scratch. " - "Note that this option recursively deletes all files " - "in the build directory, so use with care." - "Project configuration is not deleted.") + "help": ( + "Delete the entire build directory contents. " + "This includes all CMake configuration output." + "The next time the project is built, " + "CMake will configure it from scratch. " + "Note that this option recursively deletes all files " + "in the build directory, so use with care." + "Project configuration is not deleted.") }, } } diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index bde47bd41..83f5b3f1a 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -126,9 +126,9 @@ def _detect_cmake_generator(prog_name): """ Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found. """ - for (generator, _, version_check, _) in GENERATORS: - if executable_exists(version_check): - return generator + for (generator_name, generator) in GENERATORS.items(): + if executable_exists(generator["version"]): + return generator_name raise FatalError("To use %s, either the 'ninja' or 'GNU make' build tool must be available in the PATH" % prog_name) From 87c979ae0956d905a0094caff09beb390e6a2a46 Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Wed, 20 Nov 2019 12:23:03 +0100 Subject: [PATCH 2/2] docs: add idf.py fallback commands and subcommand options --- docs/en/api-guides/build-system.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/en/api-guides/build-system.rst b/docs/en/api-guides/build-system.rst index 580d14e9f..fa9bffb71 100644 --- a/docs/en/api-guides/build-system.rst +++ b/docs/en/api-guides/build-system.rst @@ -67,7 +67,7 @@ The :ref:`getting started guide ` contains a brief introd ``idf.py`` should be run in an ESP-IDF "project" directory, ie one containing a ``CMakeLists.txt`` file. Older style projects with a Makefile will not work with ``idf.py``. -Type ``idf.py --help`` for a full list of commands. Here are a summary of the most useful ones: +Type ``idf.py --help`` for a list of commands. Here are a summary of the most useful ones: - ``idf.py menuconfig`` runs the "menuconfig" tool to configure the project. - ``idf.py build`` will build the project found in the current directory. This can involve multiple steps: @@ -84,6 +84,8 @@ Type ``idf.py --help`` for a full list of commands. Here are a summary of the mo Multiple ``idf.py`` commands can be combined into one. For example, ``idf.py -p COM4 clean flash monitor`` will clean the source tree, then build the project and flash it to the ESP32 before running the serial monitor. +For commands that are not known to ``idf.py`` an attempt to execute them as a build system target will be made. + .. note:: The environment variables ``ESPPORT`` and ``ESPBAUD`` can be used to set default values for the ``-p`` and ``-b`` options, respectively. Providing these options on the command line overrides the default. .. _idf.py-size: @@ -101,8 +103,8 @@ The order of multiple ``idf.py`` commands on the same invocation is not importan idf.py options ^^^^^^^^^^^^^^ - -To list all available options, run ``idf.py --help``. +To list all available root level options, run ``idf.py --help``. To list options that are specific for a subcommand, run ``idf.py --help``, for example ``idf.py monitor --help``. +Here is a list of some useful options: - ``-C `` allows overriding the project directory from the default current working directory. - ``-B `` allows overriding the build directory from the default ``build`` subdirectory of the project directory.