ci: reimplement build_examples_cmake using {find,build}_apps.py
This commit is contained in:
Ivan Grokhotkov
parent
0e6e7f49be
commit
e1aee5ad09
|
|
@ -1,24 +1,10 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Build all examples from the examples directory, out of tree to
|
||||
# Build all examples from the examples directory, in BUILD_PATH to
|
||||
# ensure they can run when copied to a new directory.
|
||||
#
|
||||
# Runs as part of CI process.
|
||||
#
|
||||
# Assumes PWD is an out-of-tree build directory, and will copy examples
|
||||
# to individual subdirectories, one by one.
|
||||
#
|
||||
#
|
||||
# Without arguments it just builds all examples
|
||||
#
|
||||
# With one argument <JOB_NAME> it builds part of the examples. This is a useful for
|
||||
# parallel execution in CI.
|
||||
# <JOB_NAME> must look like this:
|
||||
# <some_text_label>_<num>
|
||||
# It scans .gitlab-ci.yaml to count number of jobs which have name "<some_text_label>_<num>"
|
||||
# It scans the filesystem to count all examples
|
||||
# Based on this, it decides to run qa set of examples.
|
||||
#
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
|
||||
|
|
@ -31,9 +17,6 @@ fi
|
|||
set -o errexit # Exit if command failed.
|
||||
set -o pipefail # Exit if pipe failed.
|
||||
|
||||
# Remove the initial space and instead use '\n'.
|
||||
IFS=$'\n\t'
|
||||
|
||||
export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -45,151 +28,77 @@ die() {
|
|||
|
||||
[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
|
||||
[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
|
||||
[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
|
||||
[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
|
||||
[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
|
||||
[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
|
||||
|
||||
if [ -z ${CI_NODE_TOTAL} ]; then
|
||||
CI_NODE_TOTAL=1
|
||||
echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
|
||||
fi
|
||||
if [ -z ${CI_NODE_INDEX} ]; then
|
||||
# Gitlab uses a 1-based index
|
||||
CI_NODE_INDEX=1
|
||||
echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
|
||||
fi
|
||||
|
||||
|
||||
set -o nounset # Exit if variable not set.
|
||||
|
||||
echo "build_examples running in ${PWD} for target $IDF_TARGET"
|
||||
|
||||
# only 0 or 1 arguments
|
||||
[ $# -le 1 ] || die "Have to run as $(basename $0) [<START_EXAMPLE_NUMBER>]"
|
||||
|
||||
export BATCH_BUILD=1
|
||||
export V=0 # only build verbose if there's an error
|
||||
|
||||
shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024)
|
||||
|
||||
RESULT=0
|
||||
FAILED_EXAMPLES=""
|
||||
RESULT_ISSUES=22 # magic number result code for issues found
|
||||
LOG_SUSPECTED=${LOG_PATH}/common_log.txt
|
||||
touch ${LOG_SUSPECTED}
|
||||
SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
|
||||
|
||||
EXAMPLE_PATHS=$( get_supported_examples.sh $IDF_TARGET | sed "s#^#${IDF_PATH}\/examples\/#g" | awk '{print $0"/CmakeLists.txt"}' )
|
||||
NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l )
|
||||
# just a plausibility check
|
||||
[ ${NUM_OF_EXAMPLES} -lt 50 ] && die "NUM_OF_EXAMPLES is bad"
|
||||
|
||||
echo "All examples found for target $IDF_TARGET:"
|
||||
echo $EXAMPLE_PATHS
|
||||
echo "Number of examples: $NUM_OF_EXAMPLES"
|
||||
|
||||
if [ -z "${CI_NODE_TOTAL:-}" ]
|
||||
then
|
||||
START_NUM=0
|
||||
if [ "${1:-}" ]; then
|
||||
START_NUM=$1
|
||||
fi
|
||||
END_NUM=${NUM_OF_EXAMPLES}
|
||||
else
|
||||
JOB_NUM=${CI_NODE_INDEX}
|
||||
# count number of the jobs
|
||||
NUM_OF_JOBS=${CI_NODE_TOTAL}
|
||||
|
||||
# separate intervals
|
||||
#57 / 5 == 12
|
||||
NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
|
||||
[ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
|
||||
|
||||
# ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
|
||||
START_NUM=$(( (${JOB_NUM} - 1) * ${NUM_OF_EX_PER_JOB} ))
|
||||
[ -z ${START_NUM} ] && die "START_NUM is bad"
|
||||
|
||||
END_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
|
||||
[ -z ${END_NUM} ] && die "END_NUM is bad"
|
||||
export REALPATH=realpath
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
export REALPATH=grealpath
|
||||
fi
|
||||
|
||||
build_example () {
|
||||
local ID=$1
|
||||
shift
|
||||
local CMAKELISTS=$1
|
||||
shift
|
||||
# Convert LOG_PATH and BUILD_PATH to relative, to make the json file less verbose.
|
||||
LOG_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${LOG_PATH})
|
||||
BUILD_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${BUILD_PATH})
|
||||
|
||||
local EXAMPLE_DIR=$(dirname "${CMAKELISTS}")
|
||||
local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
|
||||
ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
|
||||
JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
|
||||
mkdir -p "${BUILD_PATH}/example_builds"
|
||||
|
||||
echo "Building ${EXAMPLE_NAME} for ${IDF_TARGET} as ${ID}..."
|
||||
mkdir -p "example_builds/${IDF_TARGET}/${ID}"
|
||||
cp -r "${EXAMPLE_DIR}" "example_builds/${IDF_TARGET}/${ID}"
|
||||
pushd "example_builds/${IDF_TARGET}/${ID}/${EXAMPLE_NAME}"
|
||||
# be stricter in the CI build than the default IDF settings
|
||||
export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
|
||||
export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
|
||||
echo "build_examples running for target $IDF_TARGET"
|
||||
|
||||
# sdkconfig files are normally not checked into git, but may be present when
|
||||
# a developer runs this script locally
|
||||
rm -f sdkconfig
|
||||
cd ${IDF_PATH}
|
||||
|
||||
# If sdkconfig.ci file is present, append it to sdkconfig.defaults,
|
||||
# replacing environment variables
|
||||
if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then
|
||||
cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults
|
||||
fi
|
||||
# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage
|
||||
# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
|
||||
|
||||
# build non-verbose first
|
||||
local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
|
||||
touch ${BUILDLOG}
|
||||
# If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
|
||||
|
||||
if [ "$EXAMPLE_NAME" != "idf_as_lib" ]; then
|
||||
idf.py fullclean >>${BUILDLOG} 2>&1 &&
|
||||
idf.py build >>${BUILDLOG} 2>&1
|
||||
else
|
||||
rm -rf build &&
|
||||
./build-esp32.sh >>${BUILDLOG} 2>&1
|
||||
fi ||
|
||||
{
|
||||
RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
|
||||
}
|
||||
${IDF_PATH}/tools/find_apps.py examples \
|
||||
-vv \
|
||||
--format json \
|
||||
--build-system cmake \
|
||||
--target ${IDF_TARGET} \
|
||||
--recursive \
|
||||
--exclude examples/build_system/idf_as_lib \
|
||||
--work-dir "${BUILD_PATH}/@f/@w/@t" \
|
||||
--build-dir build \
|
||||
--build-log "${LOG_PATH}/@f.txt" \
|
||||
--output ${ALL_BUILD_LIST_JSON} \
|
||||
--config 'sdkconfig.ci=default' \
|
||||
--config 'sdkconfig.ci.*=' \
|
||||
--config '=default' \
|
||||
|
||||
cat ${BUILDLOG}
|
||||
popd
|
||||
# --config rules above explained:
|
||||
# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
|
||||
# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
|
||||
# 3. If none of the above exist, build the default configuration under the name "default"
|
||||
|
||||
grep -i "error\|warning" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || :
|
||||
}
|
||||
# The part below is where the actual builds happen
|
||||
|
||||
EXAMPLE_NUM=0
|
||||
${IDF_PATH}/tools/build_apps.py \
|
||||
-vv \
|
||||
--format json \
|
||||
--keep-going \
|
||||
--parallel-count ${CI_NODE_TOTAL} \
|
||||
--parallel-index ${CI_NODE_INDEX} \
|
||||
--output-build-list ${JOB_BUILD_LIST_JSON} \
|
||||
${ALL_BUILD_LIST_JSON}\
|
||||
|
||||
echo "Current job will build example ${START_NUM} - ${END_NUM}"
|
||||
|
||||
for EXAMPLE_PATH in ${EXAMPLE_PATHS}
|
||||
do
|
||||
if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
|
||||
then
|
||||
EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
|
||||
continue
|
||||
fi
|
||||
echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH"
|
||||
|
||||
build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}"
|
||||
|
||||
EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
|
||||
done
|
||||
|
||||
# show warnings
|
||||
echo -e "\nFound issues:"
|
||||
|
||||
# Ignore the next messages:
|
||||
# "error.o" or "-Werror" in compiler's command line
|
||||
# "reassigning to symbol" or "changes choice state" in sdkconfig
|
||||
# 'Compiler and toochain versions is not supported' from crosstool_version_check.cmake
|
||||
IGNORE_WARNS="\
|
||||
library/error\.o\
|
||||
\|.*error.*\.c\.obj\
|
||||
\|\ -Werror\
|
||||
\|error\.d\
|
||||
\|reassigning to symbol\
|
||||
\|changes choice state\
|
||||
\|crosstool_version_check\.cmake\
|
||||
"
|
||||
|
||||
sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \
|
||||
&& RESULT=$RESULT_ISSUES \
|
||||
|| echo -e "\tNone"
|
||||
|
||||
[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES"
|
||||
[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!"
|
||||
|
||||
echo -e "\nReturn code = $RESULT"
|
||||
|
||||
exit $RESULT
|
||||
# Check for build warnings
|
||||
${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}
|
||||
|
|
|
|||
|
|
@ -197,17 +197,20 @@ build_examples_make:
|
|||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- build_examples/*/*/*/*/build/*.bin
|
||||
- build_examples/*/*/*/*/sdkconfig
|
||||
- build_examples/*/*/*/*/build/*.elf
|
||||
- build_examples/*/*/*/*/build/*.map
|
||||
- build_examples/*/*/*/*/build/flasher_args.json
|
||||
- build_examples/*/*/*/*/build/bootloader/*.bin
|
||||
- build_examples/*/*/*/*/build/partition_table/*.bin
|
||||
- build_examples/list.json
|
||||
- build_examples/list_job_*.json
|
||||
- build_examples/*/*/*/sdkconfig
|
||||
- build_examples/*/*/*/build/*.bin
|
||||
- build_examples/*/*/*/build/*.elf
|
||||
- build_examples/*/*/*/build/*.map
|
||||
- build_examples/*/*/*/build/flasher_args.json
|
||||
- build_examples/*/*/*/build/bootloader/*.bin
|
||||
- build_examples/*/*/*/build/partition_table/*.bin
|
||||
- $LOG_PATH
|
||||
expire_in: 3 days
|
||||
variables:
|
||||
LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake"
|
||||
LOG_PATH: "$CI_PROJECT_DIR/log_examples"
|
||||
BUILD_PATH: "$CI_PROJECT_DIR/build_examples"
|
||||
only:
|
||||
variables:
|
||||
- $BOT_TRIGGER_WITH_LABEL == null
|
||||
|
|
@ -218,10 +221,7 @@ build_examples_make:
|
|||
script:
|
||||
# it's not possible to build 100% out-of-tree and have the "artifacts"
|
||||
# mechanism work, but this is the next best thing
|
||||
- rm -rf build_examples
|
||||
- mkdir build_examples
|
||||
- cd build_examples
|
||||
# build some of examples
|
||||
- mkdir -p ${BUILD_PATH}
|
||||
- mkdir -p ${LOG_PATH}
|
||||
- ${IDF_PATH}/tools/ci/build_examples_cmake.sh
|
||||
# Check if the tests demand CMake built binaries. If not, delete them
|
||||
|
|
|
|||
Loading…
Reference in a new issue