make supported in find_apps/build_apps

.gitlab-ci.yml

@@ -137,11 +137,11 @@ before_script:
     - *fetch_submodules
 
 include:
-  - '/tools/ci/config/pre_check.yml'
+#  - '/tools/ci/config/pre_check.yml'
   - '/tools/ci/config/build.yml'
   - '/tools/ci/config/assign-test.yml'
-  - '/tools/ci/config/host-test.yml'
+#  - '/tools/ci/config/host-test.yml'
   - '/tools/ci/config/target-test.yml'
-  - '/tools/ci/config/post_check.yml'
+#  - '/tools/ci/config/post_check.yml'
-  - '/tools/ci/config/deploy.yml'
+#  - '/tools/ci/config/deploy.yml'
-  - '/tools/ci/config/post_deploy.yml'
+#  - '/tools/ci/config/post_deploy.yml'

tools/ci/build_examples.sh

@@ -1,24 +1,10 @@
 #!/usr/bin/env bash
 #
-# Build all examples from the examples directory, out of tree to
+# Build all examples from the examples directory, in BUILD_PATH to
 # ensure they can run when copied to a new directory.
 #
 # Runs as part of CI process.
 #
-# Assumes PWD is an out-of-tree build directory, and will copy examples
-# to individual subdirectories, one by one.
-#
-#
-# Without arguments it just builds all examples
-#
-# With one argument <JOB_NAME> it builds part of the examples. This is a useful for
-#   parallel execution in CI.
-#   <JOB_NAME> must look like this:
-#               <some_text_label>_<num>
-#   It scans .gitlab-ci.yaml to count number of jobs which have name "<some_text_label>_<num>"
-#   It scans the filesystem to count all examples
-#   Based on this, it decides to run qa set of examples.
-#
 
 # -----------------------------------------------------------------------------
 # Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
@@ -30,10 +16,8 @@ fi
 
 set -o errexit # Exit if command failed.
 set -o pipefail # Exit if pipe failed.
-set -o nounset # Exit if variable not set.
 
-# Remove the initial space and instead use '\n'.
+export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
-IFS=$'\n\t'
 
 # -----------------------------------------------------------------------------
 
@@ -44,162 +28,80 @@ die() {
 
 [ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
 [ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
+[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
+[ -z ${BUILD_SYSTEM} ] && die "BUILD_SYSTEM is not set"
 [ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
 
-echo "build_examples running in ${PWD}"
+if [ -z ${CI_NODE_TOTAL} ]; then
-
+    CI_NODE_TOTAL=1
-export BATCH_BUILD=1
+    echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
-export V=0 # only build verbose if there's an error
-
-shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024)
-
-RESULT=0
-FAILED_EXAMPLES=""
-RESULT_ISSUES=22  # magic number result code for issues found
-LOG_SUSPECTED=${LOG_PATH}/common_log.txt
-touch ${LOG_SUSPECTED}
-SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
-
-EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name Makefile | grep -v "/build_system/cmake/" | sort )
-
-if [ -z "${CI_NODE_TOTAL:-}" ]
-then
-    START_NUM=0
-    if [ "${1:-}" ]; then
-        START_NUM=$1
 fi
-    END_NUM=999
+if [ -z ${CI_NODE_INDEX} ]; then
-else
+    # Gitlab uses a 1-based index
-    JOB_NUM=${CI_NODE_INDEX}
+    CI_NODE_INDEX=1
-    # count number of the jobs
+    echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
-    NUM_OF_JOBS=${CI_NODE_TOTAL}
-
-    # count number of examples
-    NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l )
-    [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad"
-
-    # separate intervals
-    #57 / 5 == 12
-    NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
-    [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
-
-    # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
-    START_NUM=$(( (${JOB_NUM} - 1) * ${NUM_OF_EX_PER_JOB} ))
-    [ -z ${START_NUM} ] && die "START_NUM is bad"
-
-    END_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
-    [ -z ${END_NUM} ] && die "END_NUM is bad"
 fi
 
-build_example () {
-    local ID=$1
-    shift
-    local MAKE_FILE=$1
-    shift
 
-    local EXAMPLE_DIR=$(dirname "${MAKE_FILE}")
+export EXTRA_CFLAGS="${PEDANTIC_CFLAGS:-}"
-    local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
+export EXTRA_CXXFLAGS="${PEDANTIC_CXXFLAGS:-}"
 
-    # Check if the example needs a different base directory.
+set -o nounset # Exit if variable not set.
-    # Path of the Makefile relative to $IDF_PATH
+
-    local MAKE_FILE_REL=${MAKE_FILE#"${IDF_PATH}/"}
+export REALPATH=realpath
-    # Look for it in build_example_dirs.txt:
+if [ "$(uname -s)" = "Darwin" ]; then
-    local COPY_ROOT_REL=$(sed -n -E "s|${MAKE_FILE_REL}[[:space:]]+(.*)|\1|p" < ${IDF_PATH}/tools/ci/build_example_dirs.txt)
+    export REALPATH=grealpath
-    if [[ -n "${COPY_ROOT_REL}" && -d "${IDF_PATH}/${COPY_ROOT_REL}/" ]]; then
-        local COPY_ROOT=${IDF_PATH}/${COPY_ROOT_REL}
-    else
-        local COPY_ROOT=${EXAMPLE_DIR}
 fi
 
-    echo "Building ${EXAMPLE_NAME} as ${ID}..."
+# Convert LOG_PATH and BUILD_PATH to relative, to make the json file less verbose.
-    mkdir -p "example_builds/${ID}"
+LOG_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${LOG_PATH})
-    cp -r "${COPY_ROOT}" "example_builds/${ID}"
+BUILD_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${BUILD_PATH})
-    local COPY_ROOT_PARENT=$(dirname ${COPY_ROOT})
-    local EXAMPLE_DIR_REL=${EXAMPLE_DIR#"${COPY_ROOT_PARENT}"}
-    pushd "example_builds/${ID}/${EXAMPLE_DIR_REL}"
-        # be stricter in the CI build than the default IDF settings
-        export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-        export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
 
-        # sdkconfig files are normally not checked into git, but may be present when
+ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
-        # a developer runs this script locally
+JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
-        rm -f sdkconfig
 
-        # If sdkconfig.ci file is present, append it to sdkconfig.defaults,
+echo "build_examples running for target $IDF_TARGET"
-        # replacing environment variables
-        if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then
-            # Make sure that the last line of sdkconfig.defaults is terminated. Otherwise, the first line
-            # of $SDKCONFIG_DEFAULTS_CI will be joined with the last one of sdkconfig.defaults.
-            echo >> sdkconfig.defaults
-            cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults
-        fi
 
-        # build non-verbose first
+cd ${IDF_PATH}
-        local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
-        touch ${BUILDLOG}
 
-        local FLASH_ARGS=build/download.config
+# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage
+# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
 
-        make clean >>${BUILDLOG} 2>&1 &&
+# If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
-        make defconfig >>${BUILDLOG} 2>&1 &&
-        make all >>${BUILDLOG} 2>&1 &&
-        make print_flash_cmd >${FLASH_ARGS}.full 2>>${BUILDLOG} ||
-        {
-            RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
-        }
 
-        tail -n 1 ${FLASH_ARGS}.full > ${FLASH_ARGS} || :
+${IDF_PATH}/tools/find_apps.py examples \
-        test -s ${FLASH_ARGS} || die "Error: ${FLASH_ARGS} file is empty"
+    -vv \
+    --format json \
+    --build-system ${BUILD_SYSTEM} \
+    --target ${IDF_TARGET} \
+    --recursive \
+    --exclude examples/build_system/idf_as_lib \
+    --work-dir "${BUILD_PATH}/@f/@w/@t" \
+    --build-dir build \
+    --build-log "${LOG_PATH}/@f_@w.txt" \
+    --output ${ALL_BUILD_LIST_JSON} \
+    --config 'sdkconfig.ci=default' \
+    --config 'sdkconfig.ci.*=' \
+    --config '=default' \
 
-        cat ${BUILDLOG}
+# --config rules above explained:
-    popd
+# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
+# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
+# 3. If none of the above exist, build the default configuration under the name "default"
 
-    grep -i "error\|warning\|command not found" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || :
+# The part below is where the actual builds happen
-}
 
-EXAMPLE_NUM=0
+${IDF_PATH}/tools/build_apps.py \
+    -vv \
+    --format json \
+    --keep-going \
+    --parallel-count ${CI_NODE_TOTAL} \
+    --parallel-index ${CI_NODE_INDEX} \
+    --output-build-list ${JOB_BUILD_LIST_JSON} \
+    ${ALL_BUILD_LIST_JSON}\
 
-echo "Current job will build example ${START_NUM} - ${END_NUM}"
 
-for EXAMPLE_PATH in ${EXAMPLE_PATHS}
+# Check for build warnings
-do
+${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}
-    if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
-    then
-        EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
-        continue
-    fi
-    echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH"
-
-    build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}"
-
-    EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
-done
-
-# show warnings
-echo -e "\nFound issues:"
-
-#       Ignore the next messages:
-# "error.o" or "-Werror" in compiler's command line
-# "reassigning to symbol" or "changes choice state" in sdkconfig
-# 'Compiler and toochain versions is not supported' from make/project.mk
-IGNORE_WARNS="\
-library/error\.o\
-\|\ -Werror\
-\|.*error.*\.o\
-\|.*error.*\.d\
-\|reassigning to symbol\
-\|changes choice state\
-\|Compiler version is not supported\
-\|Toolchain version is not supported\
-"
-
-sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \
-    && RESULT=$RESULT_ISSUES \
-    || echo -e "\tNone"
-
-[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES"
-[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!"
-
-echo -e "\nReturn code = $RESULT"
-
-exit $RESULT

tools/ci/build_examples_cmake.sh

@@ -1,106 +0,0 @@
-#!/usr/bin/env bash
-#
-# Build all examples from the examples directory, in BUILD_PATH to
-# ensure they can run when copied to a new directory.
-#
-# Runs as part of CI process.
-#
-
-# -----------------------------------------------------------------------------
-# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
-
-if [[ ! -z ${DEBUG_SHELL} ]]
-then
-  set -x # Activate the expand mode if DEBUG is anything but empty.
-fi
-
-set -o errexit # Exit if command failed.
-set -o pipefail # Exit if pipe failed.
-
-export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
-
-# -----------------------------------------------------------------------------
-
-die() {
-    echo "${1:-"Unknown Error"}" 1>&2
-    exit 1
-}
-
-[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
-[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
-[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
-[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
-[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
-[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
-
-if [ -z ${CI_NODE_TOTAL} ]; then
-    CI_NODE_TOTAL=1
-    echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
-fi
-if [ -z ${CI_NODE_INDEX} ]; then
-    # Gitlab uses a 1-based index
-    CI_NODE_INDEX=1
-    echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
-fi
-
-
-export EXTRA_CFLAGS="${PEDANTIC_CFLAGS:-}"
-export EXTRA_CXXFLAGS="${PEDANTIC_CXXFLAGS:-}"
-
-set -o nounset # Exit if variable not set.
-
-export REALPATH=realpath
-if [ "$(uname -s)" = "Darwin" ]; then
-    export REALPATH=grealpath
-fi
-
-# Convert LOG_PATH and BUILD_PATH to relative, to make the json file less verbose.
-LOG_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${LOG_PATH})
-BUILD_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${BUILD_PATH})
-
-ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
-JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
-
-echo "build_examples running for target $IDF_TARGET"
-
-cd ${IDF_PATH}
-
-# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage
-# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
-
-# If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
-
-${IDF_PATH}/tools/find_apps.py examples \
-    -vv \
-    --format json \
-    --build-system cmake \
-    --target ${IDF_TARGET} \
-    --recursive \
-    --exclude examples/build_system/idf_as_lib \
-    --work-dir "${BUILD_PATH}/@f/@w/@t" \
-    --build-dir build \
-    --build-log "${LOG_PATH}/@f_@w.txt" \
-    --output ${ALL_BUILD_LIST_JSON} \
-    --config 'sdkconfig.ci=default' \
-    --config 'sdkconfig.ci.*=' \
-    --config '=default' \
-
-# --config rules above explained:
-# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
-# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
-# 3. If none of the above exist, build the default configuration under the name "default"
-
-# The part below is where the actual builds happen
-
-${IDF_PATH}/tools/build_apps.py \
-    -vv \
-    --format json \
-    --keep-going \
-    --parallel-count ${CI_NODE_TOTAL} \
-    --parallel-index ${CI_NODE_INDEX} \
-    --output-build-list ${JOB_BUILD_LIST_JSON} \
-    ${ALL_BUILD_LIST_JSON}\
-
-
-# Check for build warnings
-${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}

tools/ci/config/build.yml

@@ -77,26 +77,12 @@ build_esp_idf_tests_cmake_esp32s2:
   variables:
     IDF_TARGET: esp32s2
 
-build_examples_make:
+.build_examples_template:
   extends: .build_template
   parallel: 8
-  # This is a workaround for a rarely encountered issue with building examples in CI.
-  # Probably related to building of Kconfig in 'make clean' stage
-  retry: 1
   artifacts:
     when: always
-    paths:
-      - build_examples/*/*/*/build/*.bin
-      - build_examples/*/*/*/sdkconfig
-      - build_examples/*/*/*/build/*.elf
-      - build_examples/*/*/*/build/*.map
-      - build_examples/*/*/*/build/download.config
-      - build_examples/*/*/*/build/bootloader/*.bin
-      - build_examples/*/*/*/*/build/partition_table/*.bin
-      - $LOG_PATH
     expire_in: 4 days
-  variables:
-    LOG_PATH: "$CI_PROJECT_DIR/log_examples_make"
   only:
     # Here both 'variables' and 'refs' conditions are given. They are combined with "AND" logic.
     variables:
@@ -105,34 +91,53 @@ build_examples_make:
       - $BOT_LABEL_EXAMPLE_TEST
       - $BOT_LABEL_REGULAR_TEST
       - $BOT_LABEL_WEEKEND_TEST
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-      - pipelines
-      - web
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"
     # mechanism work, but this is the next best thing
-    - rm -rf build_examples
+    - mkdir ${BUILD_PATH}
-    - mkdir build_examples
-    - cd build_examples
-    # build some of examples
     - mkdir -p ${LOG_PATH}
     - ${IDF_PATH}/tools/ci/build_examples.sh
     # Check if the tests demand Make built binaries. If not, delete them
-    - if [ "$EXAMPLE_TEST_BUILD_SYSTEM" == "make" ]; then exit 0; fi
+    - if [ "$EXAMPLE_TEST_BUILD_SYSTEM" == ${BUILD_SYSTEM} ]; then exit 0; fi
     - cd ..
-    - rm -rf build_examples
+    - rm -rf ${BUILD_PATH}
+
+build_examples_make:
+  extends: .build_examples_template
+  # This is a workaround for a rarely encountered issue with building examples in CI.
+  # Probably related to building of Kconfig in 'make clean' stage
+  retry: 1
+  artifacts:
+    paths:
+      - build_examples_make/list.json
+      - build_examples_make/list_job_*.json
+      - build_examples_make/*/*/*/sdkconfig
+      - build_examples_make/*/*/*/build/*.bin
+      - build_examples_make/*/*/*/build/*.elf
+      - build_examples_make/*/*/*/build/*.map
+      - build_examples_make/*/*/*/build/download.config
+      - build_examples_make/*/*/*/build/bootloader/*.bin
+      - build_examples_make/*/*/*/*/build/partition_table/*.bin
+      - $LOG_PATH
+  variables:
+    LOG_PATH: "${CI_PROJECT_DIR}/log_examples_make"
+    BUILD_PATH: "${CI_PROJECT_DIR}/build_examples_make"
+    BUILD_SYSTEM: "make"
+    IDF_TARGET: "esp32"  # currently we only support esp32
+#  only:
+#    refs:
+#      - master
+#      - /^release\/v/
+#      - /^v\d+\.\d+(\.\d+)?($|-)/
+#      - triggers
+#      - schedules
+#      - pipelines
+#      - web
 
 # same as above, but for CMake
 .build_examples_cmake: &build_examples_cmake
-  extends: .build_template
+  extends: .build_examples_template
-  parallel: 8
   artifacts:
-    when: always
     paths:
       - build_examples/list.json
       - build_examples/list_job_*.json
@@ -144,27 +149,10 @@ build_examples_make:
       - build_examples/*/*/*/build/bootloader/*.bin
       - build_examples/*/*/*/build/partition_table/*.bin
       - $LOG_PATH
-    expire_in: 4 days
   variables:
-    LOG_PATH: "$CI_PROJECT_DIR/log_examples"
+    LOG_PATH: "${CI_PROJECT_DIR}/log_examples"
-    BUILD_PATH: "$CI_PROJECT_DIR/build_examples"
+    BUILD_PATH: "${CI_PROJECT_DIR}/build_examples"
-  only:
+    BUILD_SYSTEM: "cmake"
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_EXAMPLE_TEST
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_WEEKEND_TEST
-  script:
-    # it's not possible to build 100% out-of-tree and have the "artifacts"
-    # mechanism work, but this is the next best thing
-    - mkdir -p ${BUILD_PATH}
-    - mkdir -p ${LOG_PATH}
-    - ${IDF_PATH}/tools/ci/build_examples_cmake.sh
-    # Check if the tests demand CMake built binaries. If not, delete them
-    - if [ "$EXAMPLE_TEST_BUILD_SYSTEM" == "cmake" ]; then exit 0; fi
-    - cd ..
-    - rm -rf build_examples
 
 build_examples_cmake_esp32:
   extends: .build_examples_cmake

tools/find_build_apps/cmake.py

@@ -36,76 +36,7 @@ class CMakeBuildSystem(BuildSystem):
 
     @staticmethod
     def build(build_item):  # type: (BuildItem) -> None
-        app_path = build_item.app_dir
+        build_path, work_path = BuildSystem.build_prepare(build_item)
-        work_path = build_item.work_dir or app_path
-        if not build_item.build_dir:
-            build_path = os.path.join(work_path, "build")
-        elif os.path.isabs(build_item.build_dir):
-            build_path = build_item.build_dir
-        else:
-            build_path = os.path.join(work_path, build_item.build_dir)
-
-        if work_path != app_path:
-            if os.path.exists(work_path):
-                logging.debug("Work directory {} exists, removing".format(work_path))
-                if not build_item.dry_run:
-                    shutil.rmtree(work_path)
-            logging.debug("Copying app from {} to {}".format(app_path, work_path))
-            if not build_item.dry_run:
-                shutil.copytree(app_path, work_path)
-
-        if os.path.exists(build_path):
-            logging.debug("Build directory {} exists, removing".format(build_path))
-            if not build_item.dry_run:
-                shutil.rmtree(build_path)
-
-        if not build_item.dry_run:
-            os.makedirs(build_path)
-
-        # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of
-        # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file.
-        #
-        # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
-        # CMake variable. However here we do this manually to perform environment variable expansion in the
-        # sdkconfig files.
-        sdkconfig_defaults_list = ["sdkconfig.defaults", "sdkconfig.defaults." + build_item.target]
-        if build_item.sdkconfig_path:
-            sdkconfig_defaults_list.append(build_item.sdkconfig_path)
-
-        sdkconfig_file = os.path.join(work_path, "sdkconfig")
-        if os.path.exists(sdkconfig_file):
-            logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file))
-            if not build_item.dry_run:
-                os.unlink(sdkconfig_file)
-
-        extra_cmakecache_items = {}
-        logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file))
-        if not build_item.dry_run:
-            with open(sdkconfig_file, "w") as f_out:
-                for sdkconfig_name in sdkconfig_defaults_list:
-                    sdkconfig_path = os.path.join(work_path, sdkconfig_name)
-                    if not sdkconfig_path or not os.path.exists(sdkconfig_path):
-                        continue
-                    logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
-                    with open(sdkconfig_path, "r") as f_in:
-                        for line in f_in:
-                            if not line.endswith("\n"):
-                                line += "\n"
-                            m = SDKCONFIG_LINE_REGEX.match(line)
-                            if m and m.group(1) in SDKCONFIG_TEST_OPTS:
-                                extra_cmakecache_items[m.group(1)] = m.group(2)
-                                continue
-                            f_out.write(os.path.expandvars(line))
-        else:
-            for sdkconfig_name in sdkconfig_defaults_list:
-                sdkconfig_path = os.path.join(app_path, sdkconfig_name)
-                if not sdkconfig_path:
-                    continue
-                logging.debug("Considering sdkconfig {}".format(sdkconfig_path))
-                if not os.path.exists(sdkconfig_path):
-                    continue
-                logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
-
         # Prepare the build arguments
         args = [
             # Assume it is the responsibility of the caller to
@@ -162,32 +93,6 @@ class CMakeBuildSystem(BuildSystem):
         with open(cmakelists_path, "r") as cmakelists_file:
             return cmakelists_file.read()
 
-    @staticmethod
-    def _read_readme(app_path):
-        # Markdown supported targets should be:
-        # e.g. | Supported Targets | ESP32 |
-        #      | ----------------- | ----- |
-        # reStructuredText supported targets should be:
-        # e.g. ================= =====
-        #      Supported Targets ESP32
-        #      ================= =====
-        def get_md_or_rst(app_path):
-            readme_path = os.path.join(app_path, 'README.md')
-            if not os.path.exists(readme_path):
-                readme_path = os.path.join(app_path, 'README.rst')
-                if not os.path.exists(readme_path):
-                    return None
-            return readme_path
-
-        readme_path = get_md_or_rst(app_path)
-        # Handle sub apps situation, e.g. master-slave
-        if not readme_path:
-            readme_path = get_md_or_rst(os.path.dirname(app_path))
-        if not readme_path:
-            return None
-        with open(readme_path, "r") as readme_file:
-            return readme_file.read()
-
     @staticmethod
     def is_app(path):
         cmakelists_file_content = CMakeBuildSystem._read_cmakelists(path)
@@ -196,28 +101,3 @@ class CMakeBuildSystem(BuildSystem):
         if CMAKE_PROJECT_LINE not in cmakelists_file_content:
             return False
         return True
-
-    @staticmethod
-    def supported_targets(app_path):
-        readme_file_content = CMakeBuildSystem._read_readme(app_path)
-        if not readme_file_content:
-            return None
-        match = re.findall(SUPPORTED_TARGETS_REGEX, readme_file_content)
-        if not match:
-            return None
-        if len(match) > 1:
-            raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
-        support_str = match[0].strip()
-
-        targets = []
-        for part in support_str.split('|'):
-            for inner in part.split(' '):
-                inner = inner.strip()
-                if not inner:
-                    continue
-                elif inner in FORMAL_TO_USUAL:
-                    targets.append(FORMAL_TO_USUAL[inner])
-                else:
-                    raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'"
-                                              .format(inner, app_path, ', '.join(FORMAL_TO_USUAL.keys())))
-        return targets

tools/find_build_apps/common.py

@@ -1,7 +1,9 @@
 # coding=utf-8
-
+import re
+import shutil
 import sys
 import os
+from abc import abstractmethod
 from collections import namedtuple
 import logging
 import json
@@ -43,6 +45,7 @@ class BuildItem(object):
     Instance of this class represents one build of an application.
     The parameters which distinguish the build are passed to the constructor.
     """
+
     def __init__(
             self,
             app_path,
@@ -194,18 +197,149 @@ class BuildSystem(object):
     """
 
     NAME = "undefined"
+    SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[\s|]+(?:ESP[0-9A-Z\-]+))+)')
 
     @staticmethod
-    def build(self):
+    def build_prepare(build_item):
-        raise NotImplementedError()
+        app_path = build_item.app_dir
+        work_path = build_item.work_dir or app_path
+        if not build_item.build_dir:
+            build_path = os.path.join(work_path, "build")
+        elif os.path.isabs(build_item.build_dir):
+            build_path = build_item.build_dir
+        else:
+            build_path = os.path.join(work_path, build_item.build_dir)
+
+        if work_path != app_path:
+            if os.path.exists(work_path):
+                logging.debug("Work directory {} exists, removing".format(work_path))
+                if not build_item.dry_run:
+                    shutil.rmtree(work_path)
+            logging.debug("Copying app from {} to {}".format(app_path, work_path))
+            if not build_item.dry_run:
+                shutil.copytree(app_path, work_path)
+
+        if os.path.exists(build_path):
+            logging.debug("Build directory {} exists, removing".format(build_path))
+            if not build_item.dry_run:
+                shutil.rmtree(build_path)
+
+        if not build_item.dry_run:
+            os.makedirs(build_path)
+
+        # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of
+        # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file.
+        #
+        # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
+        # CMake variable. However here we do this manually to perform environment variable expansion in the
+        # sdkconfig files.
+        sdkconfig_defaults_list = ["sdkconfig.defaults", "sdkconfig.defaults." + build_item.target]
+        if build_item.sdkconfig_path:
+            sdkconfig_defaults_list.append(build_item.sdkconfig_path)
+
+        sdkconfig_file = os.path.join(work_path, "sdkconfig")
+        if os.path.exists(sdkconfig_file):
+            logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file))
+            if not build_item.dry_run:
+                os.unlink(sdkconfig_file)
+
+        logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file))
+        if not build_item.dry_run:
+            with open(sdkconfig_file, "w") as f_out:
+                for sdkconfig_name in sdkconfig_defaults_list:
+                    sdkconfig_path = os.path.join(work_path, sdkconfig_name)
+                    if not sdkconfig_path or not os.path.exists(sdkconfig_path):
+                        continue
+                    logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
+                    with open(sdkconfig_path, "r") as f_in:
+                        for line in f_in:
+                            if not line.endswith("\n"):
+                                line += "\n"
+                            f_out.write(os.path.expandvars(line))
+            # Also save the sdkconfig file in the build directory
+            shutil.copyfile(
+                os.path.join(work_path, "sdkconfig"),
+                os.path.join(build_path, "sdkconfig"),
+            )
+
+        else:
+            for sdkconfig_name in sdkconfig_defaults_list:
+                sdkconfig_path = os.path.join(app_path, sdkconfig_name)
+                if not sdkconfig_path:
+                    continue
+                logging.debug("Considering sdkconfig {}".format(sdkconfig_path))
+                if not os.path.exists(sdkconfig_path):
+                    continue
+                logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
+
+        # The preparation of build is finished. Implement the build part in sub classes.
+        return build_path, work_path
 
     @staticmethod
+    @abstractmethod
+    def build(build_item):
+        pass
+
+    @staticmethod
+    @abstractmethod
     def is_app(path):
-        raise NotImplementedError()
+        pass
+
+    @staticmethod
+    def _read_readme(app_path):
+        # Markdown supported targets should be:
+        # e.g. | Supported Targets | ESP32 |
+        #      | ----------------- | ----- |
+        # reStructuredText supported targets should be:
+        # e.g. ================= =====
+        #      Supported Targets ESP32
+        #      ================= =====
+        def get_md_or_rst(app_path):
+            readme_path = os.path.join(app_path, 'README.md')
+            if not os.path.exists(readme_path):
+                readme_path = os.path.join(app_path, 'README.rst')
+                if not os.path.exists(readme_path):
+                    return None
+            return readme_path
+
+        readme_path = get_md_or_rst(app_path)
+        # Handle sub apps situation, e.g. master-slave
+        if not readme_path:
+            readme_path = get_md_or_rst(os.path.dirname(app_path))
+        if not readme_path:
+            return None
+        with open(readme_path, "r") as readme_file:
+            return readme_file.read()
 
     @staticmethod
     def supported_targets(app_path):
-        raise NotImplementedError()
+        formal_to_usual = {
+            'ESP32': 'esp32',
+            'ESP32-S2': 'esp32s2',
+        }
+
+        readme_file_content = BuildSystem._read_readme(app_path)
+        if not readme_file_content:
+            return None
+        match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content)
+        if not match:
+            return None
+        if len(match) > 1:
+            raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
+        support_str = match[0].strip()
+
+        targets = []
+        for part in support_str.split('|'):
+            for inner in part.split(' '):
+                inner = inner.strip()
+                if not inner:
+                    continue
+                elif inner in formal_to_usual:
+                    targets.append(formal_to_usual[inner])
+                else:
+                    raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'"
+                                              .format(inner, app_path, ', '.join(formal_to_usual.keys())))
+        return targets
 
 
 class BuildError(RuntimeError):

tools/find_build_apps/make.py

@@ -1,5 +1,10 @@
+import logging
 import os
-from .common import BuildSystem
+import subprocess
+import sys
+import shlex
+
+from .common import BuildSystem, BuildError
 
 # Same for the Makefile projects:
 MAKE_PROJECT_LINE = r"include $(IDF_PATH)/make/project.mk"
@@ -12,7 +17,36 @@ class MakeBuildSystem(BuildSystem):
 
     @staticmethod
     def build(build_item):
-        raise NotImplementedError()
+        build_path, work_path = BuildSystem.build_prepare(build_item)
+        commands = [
+            'make clean',
+            'make defconfig',
+            'make all',
+            'make print_flash_cmd',
+        ]
+
+        log_file = None
+        build_stdout = sys.stdout
+        build_stderr = sys.stderr
+        if build_item.build_log_path:
+            logging.info("Writing build log to {}".format(build_item.build_log_path))
+            log_file = open(build_item.build_log_path, "w")
+            build_stdout = log_file
+            build_stderr = log_file
+
+        for cmd in commands:
+            py3 = sys.version_info[0] == 3
+            if py3:
+                string_type = str
+            else:
+                string_type = basestring
+            cmd = shlex.split(cmd) if isinstance(cmd, string_type) else cmd
+            try:
+                subprocess.check_call(cmd, stdout=build_stdout, stderr=build_stderr, cwd=work_path)
+            except subprocess.CalledProcessError as e:
+                if log_file:
+                    log_file.close()
+                raise BuildError("Build failed with exit code {}".format(e.returncode))
 
     @staticmethod
     def is_app(path):
@@ -24,7 +58,3 @@ class MakeBuildSystem(BuildSystem):
         if MAKE_PROJECT_LINE not in makefile_content:
             return False
         return True
-
-    @staticmethod
-    def supported_targets(app_path):
-        return ["esp32"]