Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

remove build_or_not attr.

Browse source
This commit is contained in:
Fu Hanxi 2020-06-29 18:11:49 +08:00
parent 2c2f66aa21
commit 4087152305
5 changed files with 98 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
tools/build_apps.py
View file

@ -77,10 +77,9 @@ def main():
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.error("Empty build list!")
raise SystemExit(1)
logging.warning("Empty build list")
SystemExit(0)
num_builds = len(build_items)
num_jobs = args.parallel_count
@ -109,10 +108,6 @@ def main():
failed_builds = []
for build_info in builds_for_current_job:
if not build_info.build:
logging.info("Skip building app {}".format(build_info.app_dir))
continue
logging.info("Running build {}: {}".format(build_info.index, repr(build_info)))
build_system_class = BUILD_SYSTEMS[build_info.build_system]
try:

8
tools/ci/check_build_warnings.py
View file

@ -70,16 +70,12 @@ def main():
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.error("Empty build list!")
raise SystemExit(1)
logging.warning("Empty build list")
SystemExit(0)
found_warnings = 0
for build_item in build_items:
if not build_item.build:
logging.debug("Skip checking build log for app {}".format(build_item.app_dir))
continue
if not build_item.build_log_path:
logging.debug("No log file for {}".format(build_item.work_dir))
continue

163
tools/ci/python_packages/ttfw_idf/CIScanTests.py
View file

@ -1,6 +1,7 @@
import argparse
import errno
import json
import logging
import os
import re
from collections import defaultdict
@ -14,80 +15,73 @@ VALID_TARGETS = [
'esp32s2',
]
SPECIAL_REFS = [
'master',
re.compile(r'^release/v'),
re.compile(r'^v\d+\.\d+'),
TEST_LABELS = {
'example_test': 'BOT_LABEL_EXAMPLE_TEST',
'test_apps': 'BOT_LABEL_CUSTOM_TEST',
}
BUILD_ALL_LABELS = [
'BOT_LABEL_BUILD_ALL_APPS',
'BOT_LABEL_REGULAR_TEST',
]
def _judge_build_all(args_build_all):
if args_build_all:
return True
if os.getenv('BUILD_ALL_APPS'):
return True
ref = os.getenv('CI_COMMIT_REF_NAME')
pipeline_src = os.getenv('CI_PIPELINE_SOURCE')
if not ref or not pipeline_src:
return False
# scheduled pipeline will build all
if pipeline_src == 'schedule':
return True
# master, release/v..., v1.2.3..., and will build all
for special_ref in SPECIAL_REFS:
if isinstance(special_ref, re._pattern_type):
if special_ref.match(ref):
return True
else:
if ref == special_ref:
return True
def _has_build_all_label():
for label in BUILD_ALL_LABELS:
if os.getenv(label):
return True
return False
def _judge_build_or_not(action, build_all): # type: (str, bool) -> (bool, bool)
"""
:return: (build_or_not_for_test_related_apps, build_or_not_for_non_related_apps)
"""
if build_all or _has_build_all_label() or (not os.getenv('BOT_TRIGGER_WITH_LABEL')):
logging.info('Build all apps')
return True, True
if os.getenv(TEST_LABELS[action]):
logging.info('Build test cases apps')
return True, False
else:
logging.info('Skip all')
return False, False
def output_json(apps_dict_list, target, build_system, output_dir):
output_path = os.path.join(output_dir, 'scan_{}_{}.json'.format(target.lower(), build_system))
with open(output_path, 'w') as fw:
fw.writelines([json.dumps(app) + '\n' for app in apps_dict_list])
def main():
parser = argparse.ArgumentParser(description='Scan the required build tests')
actions = parser.add_subparsers(dest='action')
common = argparse.ArgumentParser(add_help=False)
common.add_argument('paths',
parser.add_argument('test_type',
choices=TEST_LABELS.keys(),
help='Scan test type')
parser.add_argument('paths',
nargs='+',
help="One or more app paths")
common.add_argument('-b', '--build-system',
help='One or more app paths')
parser.add_argument('-b', '--build-system',
choices=BUILD_SYSTEMS.keys(),
default=BUILD_SYSTEM_CMAKE)
common.add_argument('-c', '--ci-config-file',
parser.add_argument('-c', '--ci-config-file',
required=True,
help="gitlab ci config target-test file")
common.add_argument('-o', '--output-path',
parser.add_argument('-o', '--output-path',
required=True,
help="output path of the scan result")
common.add_argument("--exclude",
parser.add_argument("--exclude",
action="append",
help="Ignore specified directory. Can be used multiple times.")
common.add_argument('--preserve', action="store_true",
help='Ignore specified directory. Can be used multiple times.')
parser.add_argument('--preserve', action="store_true",
help='add this flag to preserve artifacts for all apps')
common.add_argument('--build-all', action="store_true",
parser.add_argument('--build-all', action="store_true",
help='add this flag to build all apps')
actions.add_parser('example_test', parents=[common])
actions.add_parser('test_apps', parents=[common])
args = parser.parse_args()
test_cases = []
for path in args.paths:
if args.action == 'example_test':
assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup)
elif args.action == 'test_apps':
CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup)
else:
raise SystemExit(1) # which is impossible
test_cases.extend(assign.search_cases())
build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all)
if not os.path.exists(args.output_path):
try:
@ -96,6 +90,23 @@ def main():
if e.errno != errno.EEXIST:
raise e
if (not build_standalone_apps) and (not build_test_case_apps):
for target in VALID_TARGETS:
output_json([], target, args.build_system, args.output_path)
SystemExit(0)
test_cases = []
for path in set(args.paths):
if args.test_type == 'example_test':
assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup)
elif args.test_type == 'test_apps':
CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup)
else:
raise SystemExit(1) # which is impossible
test_cases.extend(assign.search_cases())
'''
{
<target>: {
@ -113,32 +124,37 @@ def main():
build_system = args.build_system.lower()
build_system_class = BUILD_SYSTEMS[build_system]
for target in VALID_TARGETS:
target_dict = scan_info_dict[target]
test_case_apps = target_dict['test_case_apps'] = set()
for case in test_cases:
app_dir = case.case_info['app_dir']
app_target = case.case_info['target']
if app_target.lower() != target.lower():
continue
test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower()))
exclude_apps.append(app_dir)
if build_test_case_apps:
for target in VALID_TARGETS:
target_dict = scan_info_dict[target]
test_case_apps = target_dict['test_case_apps'] = set()
for case in test_cases:
app_dir = case.case_info['app_dir']
app_target = case.case_info['target']
if app_target.lower() != target.lower():
continue
test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower()))
exclude_apps.append(app_dir)
else:
for target in VALID_TARGETS:
scan_info_dict[target]['test_case_apps'] = set()
for target in VALID_TARGETS:
target_dict = scan_info_dict[target]
standalone_apps = target_dict['standalone_apps'] = set()
for path in args.paths:
standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower()))
if build_standalone_apps:
for target in VALID_TARGETS:
target_dict = scan_info_dict[target]
standalone_apps = target_dict['standalone_apps'] = set()
for path in args.paths:
standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower()))
else:
for target in VALID_TARGETS:
scan_info_dict[target]['standalone_apps'] = set()
build_all = _judge_build_all(args.build_all)
test_case_apps_preserve_default = True if build_system == 'cmake' else False
for target in VALID_TARGETS:
apps = []
for app_dir in scan_info_dict[target]['test_case_apps']:
apps.append({
'app_dir': app_dir,
'build': True,
'build_system': args.build_system,
'target': target,
'preserve': args.preserve or test_case_apps_preserve_default
@ -146,10 +162,9 @@ def main():
for app_dir in scan_info_dict[target]['standalone_apps']:
apps.append({
'app_dir': app_dir,
'build': build_all if build_system == 'cmake' else True,
'build_system': args.build_system,
'target': target,
'preserve': (args.preserve and build_all) if build_system == 'cmake' else False
'preserve': args.preserve
})
output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system))
with open(output_path, 'w') as fw:

14
tools/find_apps.py
View file

@ -49,8 +49,8 @@ def dict_from_sdkconfig(path):
def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
build_system, config_rules, build_or_not=True, preserve_artifacts=True):
# type: (str, str, str, str, str, str, typing.List[ConfigRule], bool, bool) -> typing.List[BuildItem]
build_system, config_rules, preserve_artifacts=True):
# type: (str, str, str, str, str, str, typing.List[ConfigRule], bool) -> typing.List[BuildItem]
"""
Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
:param app_path: app directory (can be / usually will be a relative path)
@ -63,7 +63,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
a different CONFIG_IDF_TARGET value.
:param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
:param config_rules: mapping of sdkconfig file name patterns to configuration names
:param build_or_not: determine if it will build in build_apps.py
:param preserve_artifacts: determine if the built binary will be uploaded as artifacts.
:return: list of BuildItems representing build configuration of the app
"""
@ -109,7 +108,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
sdkconfig_path,
config_name,
build_system,
build_or_not,
preserve_artifacts,
))
@ -125,7 +123,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
None,
default_config_name,
build_system,
build_or_not,
preserve_artifacts,
)
]
@ -296,10 +293,10 @@ def main():
apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs]
if not apps:
logging.critical("No apps found")
raise SystemExit(1)
logging.info("Found {} apps".format(len(apps)))
logging.warning("No apps found")
SystemExit(0)
logging.info("Found {} apps".format(len(apps)))
apps.sort(key=lambda x: x["app_dir"])
# Find compatible configurations of each app, collect them as BuildItems
@ -314,7 +311,6 @@ def main():
args.target or app["target"],
args.build_system or app["build_system"],
config_rules,
app["build"],
app["preserve"],
)
logging.info("Found {} builds".format(len(build_items)))

4
tools/find_build_apps/common.py
View file

@ -71,7 +71,6 @@ class BuildItem(object):
sdkconfig_path,
config_name,
build_system,
build_or_not,
preserve_artifacts,
):
# These internal variables store the paths with environment variables and placeholders;
@ -86,7 +85,6 @@ class BuildItem(object):
self.target = target
self.build_system = build_system
self.build = build_or_not
self.preserve = preserve_artifacts
self._app_name = os.path.basename(os.path.normpath(app_path))
@ -160,7 +158,6 @@ class BuildItem(object):
"config": self.config_name,
"target": self.target,
"verbose": self.verbose,
"build": self.build,
"preserve": self.preserve,
})
@ -179,7 +176,6 @@ class BuildItem(object):
config_name=d["config"],
target=d["target"],
build_system=d["build_system"],
build_or_not=d["build"],
preserve_artifacts=d["preserve"]
)
result.verbose = d["verbose"]