diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9845ba9b3..267ef246c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -137,11 +137,11 @@ before_script: - *fetch_submodules include: - - '/tools/ci/config/pre_check.yml' +# - '/tools/ci/config/pre_check.yml' - '/tools/ci/config/build.yml' - '/tools/ci/config/assign-test.yml' - - '/tools/ci/config/host-test.yml' +# - '/tools/ci/config/host-test.yml' - '/tools/ci/config/target-test.yml' - - '/tools/ci/config/post_check.yml' - - '/tools/ci/config/deploy.yml' - - '/tools/ci/config/post_deploy.yml' +# - '/tools/ci/config/post_check.yml' +# - '/tools/ci/config/deploy.yml' +# - '/tools/ci/config/post_deploy.yml' diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index 9ddd4970e..cd6e48142 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -1,24 +1,10 @@ #!/usr/bin/env bash # -# Build all examples from the examples directory, out of tree to +# Build all examples from the examples directory, in BUILD_PATH to # ensure they can run when copied to a new directory. # # Runs as part of CI process. # -# Assumes PWD is an out-of-tree build directory, and will copy examples -# to individual subdirectories, one by one. -# -# -# Without arguments it just builds all examples -# -# With one argument it builds part of the examples. This is a useful for -# parallel execution in CI. -# must look like this: -# _ -# It scans .gitlab-ci.yaml to count number of jobs which have name "_" -# It scans the filesystem to count all examples -# Based on this, it decides to run qa set of examples. -# # ----------------------------------------------------------------------------- # Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d). @@ -30,10 +16,8 @@ fi set -o errexit # Exit if command failed. set -o pipefail # Exit if pipe failed. -set -o nounset # Exit if variable not set. -# Remove the initial space and instead use '\n'. -IFS=$'\n\t' +export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH" # ----------------------------------------------------------------------------- @@ -44,162 +28,80 @@ die() { [ -z ${IDF_PATH} ] && die "IDF_PATH is not set" [ -z ${LOG_PATH} ] && die "LOG_PATH is not set" +[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set" +[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set" +[ -z ${BUILD_SYSTEM} ] && die "BUILD_SYSTEM is not set" [ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH} +[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH} -echo "build_examples running in ${PWD}" - -export BATCH_BUILD=1 -export V=0 # only build verbose if there's an error - -shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024) - -RESULT=0 -FAILED_EXAMPLES="" -RESULT_ISSUES=22 # magic number result code for issues found -LOG_SUSPECTED=${LOG_PATH}/common_log.txt -touch ${LOG_SUSPECTED} -SDKCONFIG_DEFAULTS_CI=sdkconfig.ci - -EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name Makefile | grep -v "/build_system/cmake/" | sort ) - -if [ -z "${CI_NODE_TOTAL:-}" ] -then - START_NUM=0 - if [ "${1:-}" ]; then - START_NUM=$1 - fi - END_NUM=999 -else - JOB_NUM=${CI_NODE_INDEX} - # count number of the jobs - NUM_OF_JOBS=${CI_NODE_TOTAL} - - # count number of examples - NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l ) - [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad" - - # separate intervals - #57 / 5 == 12 - NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} )) - [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad" - - # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60) - START_NUM=$(( (${JOB_NUM} - 1) * ${NUM_OF_EX_PER_JOB} )) - [ -z ${START_NUM} ] && die "START_NUM is bad" - - END_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} )) - [ -z ${END_NUM} ] && die "END_NUM is bad" +if [ -z ${CI_NODE_TOTAL} ]; then + CI_NODE_TOTAL=1 + echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}" +fi +if [ -z ${CI_NODE_INDEX} ]; then + # Gitlab uses a 1-based index + CI_NODE_INDEX=1 + echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}" fi -build_example () { - local ID=$1 - shift - local MAKE_FILE=$1 - shift - local EXAMPLE_DIR=$(dirname "${MAKE_FILE}") - local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}") +export EXTRA_CFLAGS="${PEDANTIC_CFLAGS:-}" +export EXTRA_CXXFLAGS="${PEDANTIC_CXXFLAGS:-}" - # Check if the example needs a different base directory. - # Path of the Makefile relative to $IDF_PATH - local MAKE_FILE_REL=${MAKE_FILE#"${IDF_PATH}/"} - # Look for it in build_example_dirs.txt: - local COPY_ROOT_REL=$(sed -n -E "s|${MAKE_FILE_REL}[[:space:]]+(.*)|\1|p" < ${IDF_PATH}/tools/ci/build_example_dirs.txt) - if [[ -n "${COPY_ROOT_REL}" && -d "${IDF_PATH}/${COPY_ROOT_REL}/" ]]; then - local COPY_ROOT=${IDF_PATH}/${COPY_ROOT_REL} - else - local COPY_ROOT=${EXAMPLE_DIR} - fi +set -o nounset # Exit if variable not set. - echo "Building ${EXAMPLE_NAME} as ${ID}..." - mkdir -p "example_builds/${ID}" - cp -r "${COPY_ROOT}" "example_builds/${ID}" - local COPY_ROOT_PARENT=$(dirname ${COPY_ROOT}) - local EXAMPLE_DIR_REL=${EXAMPLE_DIR#"${COPY_ROOT_PARENT}"} - pushd "example_builds/${ID}/${EXAMPLE_DIR_REL}" - # be stricter in the CI build than the default IDF settings - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} +export REALPATH=realpath +if [ "$(uname -s)" = "Darwin" ]; then + export REALPATH=grealpath +fi - # sdkconfig files are normally not checked into git, but may be present when - # a developer runs this script locally - rm -f sdkconfig +# Convert LOG_PATH and BUILD_PATH to relative, to make the json file less verbose. +LOG_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${LOG_PATH}) +BUILD_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${BUILD_PATH}) - # If sdkconfig.ci file is present, append it to sdkconfig.defaults, - # replacing environment variables - if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then - # Make sure that the last line of sdkconfig.defaults is terminated. Otherwise, the first line - # of $SDKCONFIG_DEFAULTS_CI will be joined with the last one of sdkconfig.defaults. - echo >> sdkconfig.defaults - cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults - fi +ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json" +JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json" - # build non-verbose first - local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt - touch ${BUILDLOG} +echo "build_examples running for target $IDF_TARGET" - local FLASH_ARGS=build/download.config +cd ${IDF_PATH} - make clean >>${BUILDLOG} 2>&1 && - make defconfig >>${BUILDLOG} 2>&1 && - make all >>${BUILDLOG} 2>&1 && - make print_flash_cmd >${FLASH_ARGS}.full 2>>${BUILDLOG} || - { - RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ; - } +# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage +# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact. - tail -n 1 ${FLASH_ARGS}.full > ${FLASH_ARGS} || : - test -s ${FLASH_ARGS} || die "Error: ${FLASH_ARGS} file is empty" +# If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. - cat ${BUILDLOG} - popd +${IDF_PATH}/tools/find_apps.py examples \ + -vv \ + --format json \ + --build-system ${BUILD_SYSTEM} \ + --target ${IDF_TARGET} \ + --recursive \ + --exclude examples/build_system/idf_as_lib \ + --work-dir "${BUILD_PATH}/@f/@w/@t" \ + --build-dir build \ + --build-log "${LOG_PATH}/@f_@w.txt" \ + --output ${ALL_BUILD_LIST_JSON} \ + --config 'sdkconfig.ci=default' \ + --config 'sdkconfig.ci.*=' \ + --config '=default' \ - grep -i "error\|warning\|command not found" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || : -} +# --config rules above explained: +# 1. If sdkconfig.ci exists, use it build the example with configuration name "default" +# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration +# 3. If none of the above exist, build the default configuration under the name "default" -EXAMPLE_NUM=0 +# The part below is where the actual builds happen -echo "Current job will build example ${START_NUM} - ${END_NUM}" +${IDF_PATH}/tools/build_apps.py \ + -vv \ + --format json \ + --keep-going \ + --parallel-count ${CI_NODE_TOTAL} \ + --parallel-index ${CI_NODE_INDEX} \ + --output-build-list ${JOB_BUILD_LIST_JSON} \ + ${ALL_BUILD_LIST_JSON}\ -for EXAMPLE_PATH in ${EXAMPLE_PATHS} -do - if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]] - then - EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) - continue - fi - echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH" - build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}" - - EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) -done - -# show warnings -echo -e "\nFound issues:" - -# Ignore the next messages: -# "error.o" or "-Werror" in compiler's command line -# "reassigning to symbol" or "changes choice state" in sdkconfig -# 'Compiler and toochain versions is not supported' from make/project.mk -IGNORE_WARNS="\ -library/error\.o\ -\|\ -Werror\ -\|.*error.*\.o\ -\|.*error.*\.d\ -\|reassigning to symbol\ -\|changes choice state\ -\|Compiler version is not supported\ -\|Toolchain version is not supported\ -" - -sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \ - && RESULT=$RESULT_ISSUES \ - || echo -e "\tNone" - -[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES" -[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!" - -echo -e "\nReturn code = $RESULT" - -exit $RESULT +# Check for build warnings +${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} diff --git a/tools/ci/build_examples_cmake.sh b/tools/ci/build_examples_cmake.sh deleted file mode 100755 index ce330ab2a..000000000 --- a/tools/ci/build_examples_cmake.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -# -# Build all examples from the examples directory, in BUILD_PATH to -# ensure they can run when copied to a new directory. -# -# Runs as part of CI process. -# - -# ----------------------------------------------------------------------------- -# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d). - -if [[ ! -z ${DEBUG_SHELL} ]] -then - set -x # Activate the expand mode if DEBUG is anything but empty. -fi - -set -o errexit # Exit if command failed. -set -o pipefail # Exit if pipe failed. - -export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH" - -# ----------------------------------------------------------------------------- - -die() { - echo "${1:-"Unknown Error"}" 1>&2 - exit 1 -} - -[ -z ${IDF_PATH} ] && die "IDF_PATH is not set" -[ -z ${LOG_PATH} ] && die "LOG_PATH is not set" -[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set" -[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set" -[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH} -[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH} - -if [ -z ${CI_NODE_TOTAL} ]; then - CI_NODE_TOTAL=1 - echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}" -fi -if [ -z ${CI_NODE_INDEX} ]; then - # Gitlab uses a 1-based index - CI_NODE_INDEX=1 - echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}" -fi - - -export EXTRA_CFLAGS="${PEDANTIC_CFLAGS:-}" -export EXTRA_CXXFLAGS="${PEDANTIC_CXXFLAGS:-}" - -set -o nounset # Exit if variable not set. - -export REALPATH=realpath -if [ "$(uname -s)" = "Darwin" ]; then - export REALPATH=grealpath -fi - -# Convert LOG_PATH and BUILD_PATH to relative, to make the json file less verbose. -LOG_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${LOG_PATH}) -BUILD_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${BUILD_PATH}) - -ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json" -JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json" - -echo "build_examples running for target $IDF_TARGET" - -cd ${IDF_PATH} - -# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage -# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact. - -# If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. - -${IDF_PATH}/tools/find_apps.py examples \ - -vv \ - --format json \ - --build-system cmake \ - --target ${IDF_TARGET} \ - --recursive \ - --exclude examples/build_system/idf_as_lib \ - --work-dir "${BUILD_PATH}/@f/@w/@t" \ - --build-dir build \ - --build-log "${LOG_PATH}/@f_@w.txt" \ - --output ${ALL_BUILD_LIST_JSON} \ - --config 'sdkconfig.ci=default' \ - --config 'sdkconfig.ci.*=' \ - --config '=default' \ - -# --config rules above explained: -# 1. If sdkconfig.ci exists, use it build the example with configuration name "default" -# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration -# 3. If none of the above exist, build the default configuration under the name "default" - -# The part below is where the actual builds happen - -${IDF_PATH}/tools/build_apps.py \ - -vv \ - --format json \ - --keep-going \ - --parallel-count ${CI_NODE_TOTAL} \ - --parallel-index ${CI_NODE_INDEX} \ - --output-build-list ${JOB_BUILD_LIST_JSON} \ - ${ALL_BUILD_LIST_JSON}\ - - -# Check for build warnings -${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 9f37e029b..bcfd0c72e 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -77,26 +77,12 @@ build_esp_idf_tests_cmake_esp32s2: variables: IDF_TARGET: esp32s2 -build_examples_make: +.build_examples_template: extends: .build_template parallel: 8 - # This is a workaround for a rarely encountered issue with building examples in CI. - # Probably related to building of Kconfig in 'make clean' stage - retry: 1 artifacts: when: always - paths: - - build_examples/*/*/*/build/*.bin - - build_examples/*/*/*/sdkconfig - - build_examples/*/*/*/build/*.elf - - build_examples/*/*/*/build/*.map - - build_examples/*/*/*/build/download.config - - build_examples/*/*/*/build/bootloader/*.bin - - build_examples/*/*/*/*/build/partition_table/*.bin - - $LOG_PATH expire_in: 4 days - variables: - LOG_PATH: "$CI_PROJECT_DIR/log_examples_make" only: # Here both 'variables' and 'refs' conditions are given. They are combined with "AND" logic. variables: @@ -105,34 +91,53 @@ build_examples_make: - $BOT_LABEL_EXAMPLE_TEST - $BOT_LABEL_REGULAR_TEST - $BOT_LABEL_WEEKEND_TEST - refs: - - master - - /^release\/v/ - - /^v\d+\.\d+(\.\d+)?($|-)/ - - triggers - - schedules - - pipelines - - web script: # it's not possible to build 100% out-of-tree and have the "artifacts" # mechanism work, but this is the next best thing - - rm -rf build_examples - - mkdir build_examples - - cd build_examples - # build some of examples + - mkdir ${BUILD_PATH} - mkdir -p ${LOG_PATH} - ${IDF_PATH}/tools/ci/build_examples.sh # Check if the tests demand Make built binaries. If not, delete them - - if [ "$EXAMPLE_TEST_BUILD_SYSTEM" == "make" ]; then exit 0; fi + - if [ "$EXAMPLE_TEST_BUILD_SYSTEM" == ${BUILD_SYSTEM} ]; then exit 0; fi - cd .. - - rm -rf build_examples + - rm -rf ${BUILD_PATH} + +build_examples_make: + extends: .build_examples_template + # This is a workaround for a rarely encountered issue with building examples in CI. + # Probably related to building of Kconfig in 'make clean' stage + retry: 1 + artifacts: + paths: + - build_examples_make/list.json + - build_examples_make/list_job_*.json + - build_examples_make/*/*/*/sdkconfig + - build_examples_make/*/*/*/build/*.bin + - build_examples_make/*/*/*/build/*.elf + - build_examples_make/*/*/*/build/*.map + - build_examples_make/*/*/*/build/download.config + - build_examples_make/*/*/*/build/bootloader/*.bin + - build_examples_make/*/*/*/*/build/partition_table/*.bin + - $LOG_PATH + variables: + LOG_PATH: "${CI_PROJECT_DIR}/log_examples_make" + BUILD_PATH: "${CI_PROJECT_DIR}/build_examples_make" + BUILD_SYSTEM: "make" + IDF_TARGET: "esp32" # currently we only support esp32 +# only: +# refs: +# - master +# - /^release\/v/ +# - /^v\d+\.\d+(\.\d+)?($|-)/ +# - triggers +# - schedules +# - pipelines +# - web # same as above, but for CMake .build_examples_cmake: &build_examples_cmake - extends: .build_template - parallel: 8 + extends: .build_examples_template artifacts: - when: always paths: - build_examples/list.json - build_examples/list_job_*.json @@ -144,27 +149,10 @@ build_examples_make: - build_examples/*/*/*/build/bootloader/*.bin - build_examples/*/*/*/build/partition_table/*.bin - $LOG_PATH - expire_in: 4 days variables: - LOG_PATH: "$CI_PROJECT_DIR/log_examples" - BUILD_PATH: "$CI_PROJECT_DIR/build_examples" - only: - variables: - - $BOT_TRIGGER_WITH_LABEL == null - - $BOT_LABEL_BUILD - - $BOT_LABEL_EXAMPLE_TEST - - $BOT_LABEL_REGULAR_TEST - - $BOT_LABEL_WEEKEND_TEST - script: - # it's not possible to build 100% out-of-tree and have the "artifacts" - # mechanism work, but this is the next best thing - - mkdir -p ${BUILD_PATH} - - mkdir -p ${LOG_PATH} - - ${IDF_PATH}/tools/ci/build_examples_cmake.sh - # Check if the tests demand CMake built binaries. If not, delete them - - if [ "$EXAMPLE_TEST_BUILD_SYSTEM" == "cmake" ]; then exit 0; fi - - cd .. - - rm -rf build_examples + LOG_PATH: "${CI_PROJECT_DIR}/log_examples" + BUILD_PATH: "${CI_PROJECT_DIR}/build_examples" + BUILD_SYSTEM: "cmake" build_examples_cmake_esp32: extends: .build_examples_cmake diff --git a/tools/find_build_apps/cmake.py b/tools/find_build_apps/cmake.py index 84dbeded6..081de53ea 100644 --- a/tools/find_build_apps/cmake.py +++ b/tools/find_build_apps/cmake.py @@ -36,76 +36,7 @@ class CMakeBuildSystem(BuildSystem): @staticmethod def build(build_item): # type: (BuildItem) -> None - app_path = build_item.app_dir - work_path = build_item.work_dir or app_path - if not build_item.build_dir: - build_path = os.path.join(work_path, "build") - elif os.path.isabs(build_item.build_dir): - build_path = build_item.build_dir - else: - build_path = os.path.join(work_path, build_item.build_dir) - - if work_path != app_path: - if os.path.exists(work_path): - logging.debug("Work directory {} exists, removing".format(work_path)) - if not build_item.dry_run: - shutil.rmtree(work_path) - logging.debug("Copying app from {} to {}".format(app_path, work_path)) - if not build_item.dry_run: - shutil.copytree(app_path, work_path) - - if os.path.exists(build_path): - logging.debug("Build directory {} exists, removing".format(build_path)) - if not build_item.dry_run: - shutil.rmtree(build_path) - - if not build_item.dry_run: - os.makedirs(build_path) - - # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of - # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file. - # - # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS - # CMake variable. However here we do this manually to perform environment variable expansion in the - # sdkconfig files. - sdkconfig_defaults_list = ["sdkconfig.defaults", "sdkconfig.defaults." + build_item.target] - if build_item.sdkconfig_path: - sdkconfig_defaults_list.append(build_item.sdkconfig_path) - - sdkconfig_file = os.path.join(work_path, "sdkconfig") - if os.path.exists(sdkconfig_file): - logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file)) - if not build_item.dry_run: - os.unlink(sdkconfig_file) - - extra_cmakecache_items = {} - logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file)) - if not build_item.dry_run: - with open(sdkconfig_file, "w") as f_out: - for sdkconfig_name in sdkconfig_defaults_list: - sdkconfig_path = os.path.join(work_path, sdkconfig_name) - if not sdkconfig_path or not os.path.exists(sdkconfig_path): - continue - logging.debug("Appending {} to sdkconfig".format(sdkconfig_name)) - with open(sdkconfig_path, "r") as f_in: - for line in f_in: - if not line.endswith("\n"): - line += "\n" - m = SDKCONFIG_LINE_REGEX.match(line) - if m and m.group(1) in SDKCONFIG_TEST_OPTS: - extra_cmakecache_items[m.group(1)] = m.group(2) - continue - f_out.write(os.path.expandvars(line)) - else: - for sdkconfig_name in sdkconfig_defaults_list: - sdkconfig_path = os.path.join(app_path, sdkconfig_name) - if not sdkconfig_path: - continue - logging.debug("Considering sdkconfig {}".format(sdkconfig_path)) - if not os.path.exists(sdkconfig_path): - continue - logging.debug("Appending {} to sdkconfig".format(sdkconfig_name)) - + build_path, work_path = BuildSystem.build_prepare(build_item) # Prepare the build arguments args = [ # Assume it is the responsibility of the caller to @@ -162,32 +93,6 @@ class CMakeBuildSystem(BuildSystem): with open(cmakelists_path, "r") as cmakelists_file: return cmakelists_file.read() - @staticmethod - def _read_readme(app_path): - # Markdown supported targets should be: - # e.g. | Supported Targets | ESP32 | - # | ----------------- | ----- | - # reStructuredText supported targets should be: - # e.g. ================= ===== - # Supported Targets ESP32 - # ================= ===== - def get_md_or_rst(app_path): - readme_path = os.path.join(app_path, 'README.md') - if not os.path.exists(readme_path): - readme_path = os.path.join(app_path, 'README.rst') - if not os.path.exists(readme_path): - return None - return readme_path - - readme_path = get_md_or_rst(app_path) - # Handle sub apps situation, e.g. master-slave - if not readme_path: - readme_path = get_md_or_rst(os.path.dirname(app_path)) - if not readme_path: - return None - with open(readme_path, "r") as readme_file: - return readme_file.read() - @staticmethod def is_app(path): cmakelists_file_content = CMakeBuildSystem._read_cmakelists(path) @@ -196,28 +101,3 @@ class CMakeBuildSystem(BuildSystem): if CMAKE_PROJECT_LINE not in cmakelists_file_content: return False return True - - @staticmethod - def supported_targets(app_path): - readme_file_content = CMakeBuildSystem._read_readme(app_path) - if not readme_file_content: - return None - match = re.findall(SUPPORTED_TARGETS_REGEX, readme_file_content) - if not match: - return None - if len(match) > 1: - raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path)) - support_str = match[0].strip() - - targets = [] - for part in support_str.split('|'): - for inner in part.split(' '): - inner = inner.strip() - if not inner: - continue - elif inner in FORMAL_TO_USUAL: - targets.append(FORMAL_TO_USUAL[inner]) - else: - raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'" - .format(inner, app_path, ', '.join(FORMAL_TO_USUAL.keys()))) - return targets diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index fc4013d56..867bb0c17 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -1,7 +1,9 @@ # coding=utf-8 - +import re +import shutil import sys import os +from abc import abstractmethod from collections import namedtuple import logging import json @@ -43,6 +45,7 @@ class BuildItem(object): Instance of this class represents one build of an application. The parameters which distinguish the build are passed to the constructor. """ + def __init__( self, app_path, @@ -194,18 +197,149 @@ class BuildSystem(object): """ NAME = "undefined" + SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[\s|]+(?:ESP[0-9A-Z\-]+))+)') @staticmethod - def build(self): - raise NotImplementedError() + def build_prepare(build_item): + app_path = build_item.app_dir + work_path = build_item.work_dir or app_path + if not build_item.build_dir: + build_path = os.path.join(work_path, "build") + elif os.path.isabs(build_item.build_dir): + build_path = build_item.build_dir + else: + build_path = os.path.join(work_path, build_item.build_dir) + + if work_path != app_path: + if os.path.exists(work_path): + logging.debug("Work directory {} exists, removing".format(work_path)) + if not build_item.dry_run: + shutil.rmtree(work_path) + logging.debug("Copying app from {} to {}".format(app_path, work_path)) + if not build_item.dry_run: + shutil.copytree(app_path, work_path) + + if os.path.exists(build_path): + logging.debug("Build directory {} exists, removing".format(build_path)) + if not build_item.dry_run: + shutil.rmtree(build_path) + + if not build_item.dry_run: + os.makedirs(build_path) + + # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of + # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file. + # + # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS + # CMake variable. However here we do this manually to perform environment variable expansion in the + # sdkconfig files. + sdkconfig_defaults_list = ["sdkconfig.defaults", "sdkconfig.defaults." + build_item.target] + if build_item.sdkconfig_path: + sdkconfig_defaults_list.append(build_item.sdkconfig_path) + + sdkconfig_file = os.path.join(work_path, "sdkconfig") + if os.path.exists(sdkconfig_file): + logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file)) + if not build_item.dry_run: + os.unlink(sdkconfig_file) + + logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file)) + if not build_item.dry_run: + with open(sdkconfig_file, "w") as f_out: + for sdkconfig_name in sdkconfig_defaults_list: + sdkconfig_path = os.path.join(work_path, sdkconfig_name) + if not sdkconfig_path or not os.path.exists(sdkconfig_path): + continue + logging.debug("Appending {} to sdkconfig".format(sdkconfig_name)) + with open(sdkconfig_path, "r") as f_in: + for line in f_in: + if not line.endswith("\n"): + line += "\n" + f_out.write(os.path.expandvars(line)) + # Also save the sdkconfig file in the build directory + shutil.copyfile( + os.path.join(work_path, "sdkconfig"), + os.path.join(build_path, "sdkconfig"), + ) + + else: + for sdkconfig_name in sdkconfig_defaults_list: + sdkconfig_path = os.path.join(app_path, sdkconfig_name) + if not sdkconfig_path: + continue + logging.debug("Considering sdkconfig {}".format(sdkconfig_path)) + if not os.path.exists(sdkconfig_path): + continue + logging.debug("Appending {} to sdkconfig".format(sdkconfig_name)) + + # The preparation of build is finished. Implement the build part in sub classes. + return build_path, work_path @staticmethod + @abstractmethod + def build(build_item): + pass + + @staticmethod + @abstractmethod def is_app(path): - raise NotImplementedError() + pass + + @staticmethod + def _read_readme(app_path): + # Markdown supported targets should be: + # e.g. | Supported Targets | ESP32 | + # | ----------------- | ----- | + # reStructuredText supported targets should be: + # e.g. ================= ===== + # Supported Targets ESP32 + # ================= ===== + def get_md_or_rst(app_path): + readme_path = os.path.join(app_path, 'README.md') + if not os.path.exists(readme_path): + readme_path = os.path.join(app_path, 'README.rst') + if not os.path.exists(readme_path): + return None + return readme_path + + readme_path = get_md_or_rst(app_path) + # Handle sub apps situation, e.g. master-slave + if not readme_path: + readme_path = get_md_or_rst(os.path.dirname(app_path)) + if not readme_path: + return None + with open(readme_path, "r") as readme_file: + return readme_file.read() @staticmethod def supported_targets(app_path): - raise NotImplementedError() + formal_to_usual = { + 'ESP32': 'esp32', + 'ESP32-S2': 'esp32s2', + } + + readme_file_content = BuildSystem._read_readme(app_path) + if not readme_file_content: + return None + match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content) + if not match: + return None + if len(match) > 1: + raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path)) + support_str = match[0].strip() + + targets = [] + for part in support_str.split('|'): + for inner in part.split(' '): + inner = inner.strip() + if not inner: + continue + elif inner in formal_to_usual: + targets.append(formal_to_usual[inner]) + else: + raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'" + .format(inner, app_path, ', '.join(formal_to_usual.keys()))) + return targets class BuildError(RuntimeError): diff --git a/tools/find_build_apps/make.py b/tools/find_build_apps/make.py index d296817ba..3f8ab5c19 100644 --- a/tools/find_build_apps/make.py +++ b/tools/find_build_apps/make.py @@ -1,5 +1,10 @@ +import logging import os -from .common import BuildSystem +import subprocess +import sys +import shlex + +from .common import BuildSystem, BuildError # Same for the Makefile projects: MAKE_PROJECT_LINE = r"include $(IDF_PATH)/make/project.mk" @@ -12,7 +17,36 @@ class MakeBuildSystem(BuildSystem): @staticmethod def build(build_item): - raise NotImplementedError() + build_path, work_path = BuildSystem.build_prepare(build_item) + commands = [ + 'make clean', + 'make defconfig', + 'make all', + 'make print_flash_cmd', + ] + + log_file = None + build_stdout = sys.stdout + build_stderr = sys.stderr + if build_item.build_log_path: + logging.info("Writing build log to {}".format(build_item.build_log_path)) + log_file = open(build_item.build_log_path, "w") + build_stdout = log_file + build_stderr = log_file + + for cmd in commands: + py3 = sys.version_info[0] == 3 + if py3: + string_type = str + else: + string_type = basestring + cmd = shlex.split(cmd) if isinstance(cmd, string_type) else cmd + try: + subprocess.check_call(cmd, stdout=build_stdout, stderr=build_stderr, cwd=work_path) + except subprocess.CalledProcessError as e: + if log_file: + log_file.close() + raise BuildError("Build failed with exit code {}".format(e.returncode)) @staticmethod def is_app(path): @@ -24,7 +58,3 @@ class MakeBuildSystem(BuildSystem): if MAKE_PROJECT_LINE not in makefile_content: return False return True - - @staticmethod - def supported_targets(app_path): - return ["esp32"]