Merge branch 'bugfix/idf_py_dependent_tasks' into 'master'
idf.py: Fix execution order for dependent tasks Closes IDF-901 and IDFGH-1710 See merge request espressif/esp-idf!5859
This commit is contained in:
commit
44c89c0e9f
|
@ -187,6 +187,14 @@ test_idf_size:
|
||||||
- cd ${IDF_PATH}/tools/test_idf_size
|
- cd ${IDF_PATH}/tools/test_idf_size
|
||||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
|
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
|
||||||
|
|
||||||
|
test_idf_py:
|
||||||
|
extends: .host_test_template
|
||||||
|
variables:
|
||||||
|
LC_ALL: C.UTF-8
|
||||||
|
script:
|
||||||
|
- cd ${IDF_PATH}/tools/test_idf_py
|
||||||
|
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_py.py
|
||||||
|
|
||||||
test_idf_tools:
|
test_idf_tools:
|
||||||
extends: .host_test_template
|
extends: .host_test_template
|
||||||
script:
|
script:
|
||||||
|
|
|
@ -77,6 +77,7 @@ tools/mass_mfg/mfg_gen.py
|
||||||
tools/set-submodules-to-github.sh
|
tools/set-submodules-to-github.sh
|
||||||
tools/test_check_kconfigs.py
|
tools/test_check_kconfigs.py
|
||||||
tools/test_idf_monitor/run_test_idf_monitor.py
|
tools/test_idf_monitor/run_test_idf_monitor.py
|
||||||
|
tools/test_idf_py/test_idf_py.py
|
||||||
tools/test_idf_size/test.sh
|
tools/test_idf_size/test.sh
|
||||||
tools/test_idf_tools/test_idf_tools.py
|
tools/test_idf_tools/test_idf_tools.py
|
||||||
tools/unit-test-app/unit_test.py
|
tools/unit-test-app/unit_test.py
|
||||||
|
|
|
@ -490,16 +490,6 @@ endmenu\n" >> ${IDF_PATH}/Kconfig;
|
||||||
rm -rf esp32
|
rm -rf esp32
|
||||||
rm -rf mycomponents
|
rm -rf mycomponents
|
||||||
|
|
||||||
# idf.py global and subcommand parameters
|
|
||||||
print_status "Cannot set -D twice: for command and subcommand of idf.py (with different values)"
|
|
||||||
idf.py -DAAA=BBB build -DAAA=BBB -DCCC=EEE
|
|
||||||
if [ $? -eq 0 ]; then
|
|
||||||
failure "It shouldn't be allowed to set -D twice (globally and for subcommand) with different set of options"
|
|
||||||
fi
|
|
||||||
|
|
||||||
print_status "Can set -D twice: globally and for subcommand, only if values are the same"
|
|
||||||
idf.py -DAAA=BBB -DCCC=EEE build -DAAA=BBB -DCCC=EEE || failure "It should be allowed to set -D twice (globally and for subcommand) if values are the same"
|
|
||||||
|
|
||||||
# idf.py subcommand options, (using monitor with as example)
|
# idf.py subcommand options, (using monitor with as example)
|
||||||
print_status "Can set options to subcommands: print_filter for monitor"
|
print_status "Can set options to subcommands: print_filter for monitor"
|
||||||
mv ${IDF_PATH}/tools/idf_monitor.py ${IDF_PATH}/tools/idf_monitor.py.tmp
|
mv ${IDF_PATH}/tools/idf_monitor.py ${IDF_PATH}/tools/idf_monitor.py.tmp
|
||||||
|
|
136
tools/idf.py
136
tools/idf.py
|
@ -32,11 +32,12 @@ import locale
|
||||||
import multiprocessing
|
import multiprocessing
|
||||||
import os
|
import os
|
||||||
import os.path
|
import os.path
|
||||||
|
import platform
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import platform
|
from collections import Counter, OrderedDict
|
||||||
|
|
||||||
|
|
||||||
class FatalError(RuntimeError):
|
class FatalError(RuntimeError):
|
||||||
|
@ -422,6 +423,7 @@ def set_target(action, ctx, args, idf_target):
|
||||||
if os.path.exists(sdkconfig_path):
|
if os.path.exists(sdkconfig_path):
|
||||||
os.rename(sdkconfig_path, sdkconfig_old)
|
os.rename(sdkconfig_path, sdkconfig_old)
|
||||||
print("Set Target to: %s, new sdkconfig created. Existing sdkconfig renamed to sdkconfig.old." % idf_target)
|
print("Set Target to: %s, new sdkconfig created. Existing sdkconfig renamed to sdkconfig.old." % idf_target)
|
||||||
|
_ensure_build_directory(args, True)
|
||||||
|
|
||||||
|
|
||||||
def reconfigure(action, ctx, args):
|
def reconfigure(action, ctx, args):
|
||||||
|
@ -798,7 +800,7 @@ def init_cli(verbose_output=None):
|
||||||
class Option(click.Option):
|
class Option(click.Option):
|
||||||
"""Option that knows whether it should be global"""
|
"""Option that knows whether it should be global"""
|
||||||
|
|
||||||
def __init__(self, scope=None, deprecated=False, **kwargs):
|
def __init__(self, scope=None, deprecated=False, hidden=False, **kwargs):
|
||||||
"""
|
"""
|
||||||
Keyword arguments additional to Click's Option class:
|
Keyword arguments additional to Click's Option class:
|
||||||
|
|
||||||
|
@ -815,6 +817,7 @@ def init_cli(verbose_output=None):
|
||||||
|
|
||||||
self.deprecated = deprecated
|
self.deprecated = deprecated
|
||||||
self.scope = Scope(scope)
|
self.scope = Scope(scope)
|
||||||
|
self.hidden = hidden
|
||||||
|
|
||||||
if deprecated:
|
if deprecated:
|
||||||
deprecation = DeprecationMessage(deprecated)
|
deprecation = DeprecationMessage(deprecated)
|
||||||
|
@ -823,6 +826,13 @@ def init_cli(verbose_output=None):
|
||||||
if self.scope.is_global:
|
if self.scope.is_global:
|
||||||
self.help += " This option can be used at most once either globally, or for one subcommand."
|
self.help += " This option can be used at most once either globally, or for one subcommand."
|
||||||
|
|
||||||
|
def get_help_record(self, ctx):
|
||||||
|
# Backport "hidden" parameter to click 5.0
|
||||||
|
if self.hidden:
|
||||||
|
return
|
||||||
|
|
||||||
|
return super(Option, self).get_help_record(ctx)
|
||||||
|
|
||||||
class CLI(click.MultiCommand):
|
class CLI(click.MultiCommand):
|
||||||
"""Action list contains all actions with options available for CLI"""
|
"""Action list contains all actions with options available for CLI"""
|
||||||
|
|
||||||
|
@ -980,9 +990,18 @@ def init_cli(verbose_output=None):
|
||||||
|
|
||||||
def execute_tasks(self, tasks, **kwargs):
|
def execute_tasks(self, tasks, **kwargs):
|
||||||
ctx = click.get_current_context()
|
ctx = click.get_current_context()
|
||||||
global_args = PropertyDict(ctx.params)
|
global_args = PropertyDict(kwargs)
|
||||||
|
|
||||||
# Set propagated global options
|
# Show warning if some tasks are present several times in the list
|
||||||
|
dupplicated_tasks = sorted([item for item, count in Counter(task.name for task in tasks).items() if count > 1])
|
||||||
|
if dupplicated_tasks:
|
||||||
|
dupes = ", ".join('"%s"' % t for t in dupplicated_tasks)
|
||||||
|
print("WARNING: Command%s found in the list of commands more than once. "
|
||||||
|
% ("s %s are" % dupes if len(dupplicated_tasks) > 1 else " %s is" % dupes)
|
||||||
|
+ "Only first occurence will be executed.")
|
||||||
|
|
||||||
|
# Set propagated global options.
|
||||||
|
# These options may be set on one subcommand, but available in the list of global arguments
|
||||||
for task in tasks:
|
for task in tasks:
|
||||||
for key in list(task.action_args):
|
for key in list(task.action_args):
|
||||||
option = next((o for o in ctx.command.params if o.name == key), None)
|
option = next((o for o in ctx.command.params if o.name == key), None)
|
||||||
|
@ -1003,72 +1022,78 @@ def init_cli(verbose_output=None):
|
||||||
# Show warnings about global arguments
|
# Show warnings about global arguments
|
||||||
print_deprecation_warning(ctx)
|
print_deprecation_warning(ctx)
|
||||||
|
|
||||||
# Validate global arguments
|
# Make sure that define_cache_entry is mutable list and can be modified in callbacks
|
||||||
|
global_args.define_cache_entry = list(global_args.define_cache_entry)
|
||||||
|
|
||||||
|
# Execute all global action callback - first from idf.py itself, then from extensions
|
||||||
for action_callback in ctx.command.global_action_callbacks:
|
for action_callback in ctx.command.global_action_callbacks:
|
||||||
action_callback(ctx, global_args, tasks)
|
action_callback(ctx, global_args, tasks)
|
||||||
|
|
||||||
|
# Always show help when command is not provided
|
||||||
if not tasks:
|
if not tasks:
|
||||||
print(ctx.get_help())
|
print(ctx.get_help())
|
||||||
ctx.exit()
|
ctx.exit()
|
||||||
|
|
||||||
# Make sure that define_cache_entry is list
|
# Build full list of tasks to and deal with dependencies and order dependencies
|
||||||
global_args.define_cache_entry = list(global_args.define_cache_entry)
|
tasks_to_run = OrderedDict()
|
||||||
|
|
||||||
# Go through the task and create depended but missing tasks
|
|
||||||
all_tasks = [t.name for t in tasks]
|
|
||||||
tasks, tasks_to_handle = [], tasks
|
|
||||||
while tasks_to_handle:
|
|
||||||
task = tasks_to_handle.pop()
|
|
||||||
tasks.append(task)
|
|
||||||
for dep in task.dependencies:
|
|
||||||
if dep not in all_tasks:
|
|
||||||
print(
|
|
||||||
'Adding %s\'s dependency "%s" to list of actions'
|
|
||||||
% (task.name, dep)
|
|
||||||
)
|
|
||||||
dep_task = ctx.invoke(ctx.command.get_command(ctx, dep))
|
|
||||||
|
|
||||||
# Remove global options from dependent tasks
|
|
||||||
for key in list(dep_task.action_args):
|
|
||||||
option = next((o for o in ctx.command.params if o.name == key), None)
|
|
||||||
if option and (option.scope.is_global or option.scope.is_shared):
|
|
||||||
dep_task.action_args.pop(key)
|
|
||||||
|
|
||||||
tasks_to_handle.append(dep_task)
|
|
||||||
all_tasks.append(dep_task.name)
|
|
||||||
|
|
||||||
# very simple dependency management
|
|
||||||
completed_tasks = set()
|
|
||||||
while tasks:
|
while tasks:
|
||||||
task = tasks[0]
|
task = tasks[0]
|
||||||
tasks_dict = dict([(t.name, t) for t in tasks])
|
tasks_dict = dict([(t.name, t) for t in tasks])
|
||||||
|
|
||||||
name_with_aliases = task.name
|
dependecies_processed = True
|
||||||
if task.aliases:
|
|
||||||
name_with_aliases += " (aliases: %s)" % ", ".join(task.aliases)
|
|
||||||
|
|
||||||
ready_to_run = True
|
# If task have some dependecies they have to be executed before the task.
|
||||||
|
for dep in task.dependencies:
|
||||||
|
if dep not in tasks_to_run.keys():
|
||||||
|
# If dependent task is in the list of unprocessed tasks move to the front of the list
|
||||||
|
if dep in tasks_dict.keys():
|
||||||
|
dep_task = tasks.pop(tasks.index(tasks_dict[dep]))
|
||||||
|
# Otherwise invoke it with default set of options
|
||||||
|
# and put to the front of the list of unprocessed tasks
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
'Adding "%s"\'s dependency "%s" to list of commands with default set of options.'
|
||||||
|
% (task.name, dep)
|
||||||
|
)
|
||||||
|
dep_task = ctx.invoke(ctx.command.get_command(ctx, dep))
|
||||||
|
|
||||||
|
# Remove options with global scope from invoke tasks because they are alread in global_args
|
||||||
|
for key in list(dep_task.action_args):
|
||||||
|
option = next((o for o in ctx.command.params if o.name == key), None)
|
||||||
|
if option and (option.scope.is_global or option.scope.is_shared):
|
||||||
|
dep_task.action_args.pop(key)
|
||||||
|
|
||||||
|
tasks.insert(0, dep_task)
|
||||||
|
dependecies_processed = False
|
||||||
|
|
||||||
|
# Order only dependencies are moved to the front of the queue if they present in command list
|
||||||
for dep in task.order_dependencies:
|
for dep in task.order_dependencies:
|
||||||
if dep in tasks_dict.keys() and dep not in completed_tasks:
|
if dep in tasks_dict.keys() and dep not in tasks_to_run.keys():
|
||||||
tasks.insert(0, tasks.pop(tasks.index(tasks_dict[dep])))
|
tasks.insert(0, tasks.pop(tasks.index(tasks_dict[dep])))
|
||||||
ready_to_run = False
|
dependecies_processed = False
|
||||||
|
|
||||||
if ready_to_run:
|
if dependecies_processed:
|
||||||
|
# Remove task from list of unprocessed tasks
|
||||||
tasks.pop(0)
|
tasks.pop(0)
|
||||||
|
|
||||||
if task.name in completed_tasks:
|
# And add to the queue
|
||||||
print(
|
if task.name not in tasks_to_run.keys():
|
||||||
"Skipping action that is already done: %s"
|
tasks_to_run.update([(task.name, task)])
|
||||||
% name_with_aliases
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
print("Executing action: %s" % name_with_aliases)
|
|
||||||
task.run(ctx, global_args, task.action_args)
|
|
||||||
|
|
||||||
completed_tasks.add(task.name)
|
# Run all tasks in the queue
|
||||||
|
# when global_args.no_run is true idf.py works in idle mode and skips actual task execution
|
||||||
|
if not global_args.no_run:
|
||||||
|
for task in tasks_to_run.values():
|
||||||
|
name_with_aliases = task.name
|
||||||
|
if task.aliases:
|
||||||
|
name_with_aliases += " (aliases: %s)" % ", ".join(task.aliases)
|
||||||
|
|
||||||
self._print_closing_message(global_args, completed_tasks)
|
print("Executing action: %s" % name_with_aliases)
|
||||||
|
task.run(ctx, global_args, task.action_args)
|
||||||
|
|
||||||
|
self._print_closing_message(global_args, tasks_to_run.keys())
|
||||||
|
|
||||||
|
return tasks_to_run
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def merge_action_lists(*action_lists):
|
def merge_action_lists(*action_lists):
|
||||||
|
@ -1166,6 +1191,13 @@ def init_cli(verbose_output=None):
|
||||||
"help": "CMake generator.",
|
"help": "CMake generator.",
|
||||||
"type": click.Choice(GENERATOR_CMDS.keys()),
|
"type": click.Choice(GENERATOR_CMDS.keys()),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"names": ["--no-run"],
|
||||||
|
"help": "Only process arguments, but don't execute actions.",
|
||||||
|
"is_flag": True,
|
||||||
|
"hidden": True,
|
||||||
|
"default": False
|
||||||
|
},
|
||||||
],
|
],
|
||||||
"global_action_callbacks": [validate_root_options],
|
"global_action_callbacks": [validate_root_options],
|
||||||
}
|
}
|
||||||
|
@ -1276,7 +1308,7 @@ def init_cli(verbose_output=None):
|
||||||
+ "For example, \"idf.py -DNAME='VALUE' reconfigure\" "
|
+ "For example, \"idf.py -DNAME='VALUE' reconfigure\" "
|
||||||
+ 'can be used to set variable "NAME" in CMake cache to value "VALUE".',
|
+ 'can be used to set variable "NAME" in CMake cache to value "VALUE".',
|
||||||
"options": global_options,
|
"options": global_options,
|
||||||
"order_dependencies": ["menuconfig", "set-target", "fullclean"],
|
"order_dependencies": ["menuconfig", "fullclean"],
|
||||||
},
|
},
|
||||||
"set-target": {
|
"set-target": {
|
||||||
"callback": set_target,
|
"callback": set_target,
|
||||||
|
@ -1291,7 +1323,7 @@ def init_cli(verbose_output=None):
|
||||||
"nargs": 1,
|
"nargs": 1,
|
||||||
"type": click.Choice(SUPPORTED_TARGETS),
|
"type": click.Choice(SUPPORTED_TARGETS),
|
||||||
}],
|
}],
|
||||||
"dependencies": ["fullclean", "reconfigure"],
|
"dependencies": ["fullclean"],
|
||||||
},
|
},
|
||||||
"clean": {
|
"clean": {
|
||||||
"callback": clean,
|
"callback": clean,
|
||||||
|
|
102
tools/test_idf_py/test_idf_py.py
Executable file
102
tools/test_idf_py/test_idf_py.py
Executable file
|
@ -0,0 +1,102 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
#
|
||||||
|
# Copyright 2019 Espressif Systems (Shanghai) CO LTD
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
try:
|
||||||
|
from StringIO import StringIO
|
||||||
|
except ImportError:
|
||||||
|
from io import StringIO
|
||||||
|
|
||||||
|
try:
|
||||||
|
import idf
|
||||||
|
except ImportError:
|
||||||
|
sys.path.append('..')
|
||||||
|
import idf
|
||||||
|
|
||||||
|
|
||||||
|
class TestDependencyManagement(unittest.TestCase):
|
||||||
|
def test_dependencies(self):
|
||||||
|
result = idf.init_cli()(
|
||||||
|
args=['--no-run', 'flash'],
|
||||||
|
standalone_mode=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(['all', 'flash'], list(result.keys()))
|
||||||
|
|
||||||
|
def test_order_only_dependencies(self):
|
||||||
|
result = idf.init_cli()(
|
||||||
|
args=['--no-run', 'build', 'fullclean', 'all'],
|
||||||
|
standalone_mode=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(['fullclean', 'all'], list(result.keys()))
|
||||||
|
|
||||||
|
def test_repeated_dependencies(self):
|
||||||
|
result = idf.init_cli()(
|
||||||
|
args=['--no-run', 'fullclean', 'app', 'fullclean', 'fullclean'],
|
||||||
|
standalone_mode=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(['fullclean', 'app'], list(result.keys()))
|
||||||
|
|
||||||
|
def test_complex_case(self):
|
||||||
|
result = idf.init_cli()(
|
||||||
|
args=['--no-run', 'clean', 'monitor', 'clean', 'fullclean', 'flash'],
|
||||||
|
standalone_mode=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(['fullclean', 'clean', 'all', 'flash', 'monitor'], list(result.keys()))
|
||||||
|
|
||||||
|
def test_dupplicated_commands_warning(self):
|
||||||
|
capturedOutput = StringIO()
|
||||||
|
sys.stdout = capturedOutput
|
||||||
|
idf.init_cli()(
|
||||||
|
args=['--no-run', 'clean', 'monitor', 'build', 'clean', 'fullclean', 'all'],
|
||||||
|
standalone_mode=False,
|
||||||
|
)
|
||||||
|
sys.stdout = sys.__stdout__
|
||||||
|
self.assertIn('WARNING: Commands "all", "clean" are found in the list of commands more than once.',
|
||||||
|
capturedOutput.getvalue())
|
||||||
|
|
||||||
|
sys.stdout = capturedOutput
|
||||||
|
idf.init_cli()(
|
||||||
|
args=['--no-run', 'clean', 'clean'],
|
||||||
|
standalone_mode=False,
|
||||||
|
)
|
||||||
|
sys.stdout = sys.__stdout__
|
||||||
|
self.assertIn('WARNING: Command "clean" is found in the list of commands more than once.',
|
||||||
|
capturedOutput.getvalue())
|
||||||
|
|
||||||
|
|
||||||
|
class TestGlobalAndSubcommandParameters(unittest.TestCase):
|
||||||
|
def test_set_twice_same_value(self):
|
||||||
|
"""Can set -D twice: globally and for subcommand if values are the same"""
|
||||||
|
|
||||||
|
idf.init_cli()(
|
||||||
|
args=['--no-run', '-DAAA=BBB', '-DCCC=EEE', 'build', '-DAAA=BBB', '-DCCC=EEE'],
|
||||||
|
standalone_mode=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_set_twice_different_values(self):
|
||||||
|
"""Cannot set -D twice: for command and subcommand of idf.py (with different values)"""
|
||||||
|
|
||||||
|
with self.assertRaises(idf.FatalError):
|
||||||
|
idf.init_cli()(
|
||||||
|
args=['--no-run', '-DAAA=BBB', 'build', '-DAAA=EEE', '-DCCC=EEE'],
|
||||||
|
standalone_mode=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
Loading…
Reference in a new issue