Merge branch 'fix/ci_examples' into 'master'

CI: Improve the build_examples test job

See merge request !826

.gitlab-ci.yml

@@ -6,6 +6,14 @@ stages:
   - test_report
   - deploy
 
+variables:
+# system environment
+  # Common parameters for the 'make' during CI tests
+  MAKEFLAGS: "-j5 --no-keep-going"
+
+# IDF environment
+  IDF_PATH: "$CI_PROJECT_DIR"
+
 before_script:
   # add gitlab ssh key
   - mkdir -p ~/.ssh
@@ -30,7 +38,6 @@ build_template_app:
     - build
 
   variables:
-    IDF_PATH: "$CI_PROJECT_DIR"
     GIT_STRATEGY: clone
     BATCH_BUILD: "1"
     IDF_CI_BUILD: "1"
@@ -61,7 +68,6 @@ build_template_app:
     - build
 
   variables:
-    IDF_PATH: "$CI_PROJECT_DIR"
     GIT_STRATEGY: clone
     BATCH_BUILD: "1"
     V: "0"
@@ -108,22 +114,40 @@ build_esp_idf_tests:
     - make print_flash_cmd | tail -n 1 > build/download.config
     - python tools/UnitTestParser.py
 
-build_examples:
+.build_examples_template: &build_examples_template
   <<: *build_template
   artifacts:
     paths:
-      - build_examples/*/*/build/*.bin
+      - build_examples/*/*/*/build/*.bin
-      - build_examples/*/*/build/*.elf
+      - build_examples/*/*/*/build/*.elf
-      - build_examples/*/*/build/*.map
+      - build_examples/*/*/*/build/*.map
-      - build_examples/*/*/build/bootloader/*.bin
+      - build_examples/*/*/*/build/bootloader/*.bin
-    expire_in: 6 mos
+    expire_in: 1 week
-
+  variables:
+    IDF_CI_BUILD: "1"
+    GIT_STRATEGY: fetch
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"
     # mechanism work, but this is the next best thing
     - mkdir build_examples
     - cd build_examples
-    - ${IDF_PATH}/make/build_examples.sh -j5
+    # build some of examples
+    - ${IDF_PATH}/make/build_examples.sh "${CI_JOB_NAME}"
+
+build_examples_00:
+  <<: *build_examples_template
+
+build_examples_01:
+  <<: *build_examples_template
+
+build_examples_02:
+  <<: *build_examples_template
+
+build_examples_03:
+  <<: *build_examples_template
+
+build_examples_04:
+  <<: *build_examples_template
 
 build_docs:
   stage: build
@@ -177,8 +201,6 @@ test_build_system:
   image: $CI_DOCKER_REGISTRY/esp32-ci-env
   tags:
     - build_test
-  variables:
-    IDF_PATH: "$CI_PROJECT_DIR"
   script:
     - ./make/test_configure_ci_environment.sh
     - ./make/test_build_system.sh
@@ -371,7 +393,6 @@ assign_test:
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
     MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
     CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml"
-    IDF_PATH: "$CI_PROJECT_DIR"
 
   artifacts:
     when: always
@@ -416,7 +437,6 @@ assign_test:
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/unit_test"
     MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
     CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/unit_test/CIConfigs/$CI_JOB_NAME.yml"
-    IDF_PATH: "$CI_PROJECT_DIR"
 
 UT_001_01:
   <<: *unit_test_template

make/build_examples.sh

@@ -5,58 +5,158 @@
 #
 # Runs as part of CI process.
 #
-# Assumes CWD is an out-of-tree build directory, and will copy examples to individual subdirectories, one by one.
+# Assumes CWD is an out-of-tree build directory, and will copy examples
+# to individual subdirectories, one by one.
 #
-[ -z ${IDF_PATH} ] && echo "IDF_PATH is not set" && exit 1
+#
+# Without arguments it just builds all examples
+#
+# With one argument <JOB_NAME> it builds part of the examples. This is a useful for
+#   parallel execution in CI.
+#   <JOB_NAME> must look like this:
+#               <some_text_label>_<num>
+#   It scans .gitlab-ci.yaml to count number of jobs which have name "<some_text_label>_<num>"
+#   It scans the filesystem to count all examples
+#   Based on this, it decides to run qa set of examples.
+#
+
+# -----------------------------------------------------------------------------
+# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
+
+if [[ ! -z ${DEBUG} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+set -o nounset # Exit if variable not set.
+
+# Remove the initial space and instead use '\n'.
+IFS=$'\n\t'
+
+# -----------------------------------------------------------------------------
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
+
+# only 0 or 1 arguments
+[ $# -le 1 ] || die "Have to run as $(basename $0) [<JOB_NAME>]"
 
 export BATCH_BUILD=1
 export V=0 # only build verbose if there's an error
 
-EXAMPLE_NUM=1
 RESULT=0
 FAILED_EXAMPLES=""
 
 RESULT_WARNINGS=22  # magic number result code for "warnings found"
+LOG_WARNINGS=$(mktemp -t example_all.XXXX.log)
 
-# traverse categories
+if [ $# -eq 0 ]
-for category in ${IDF_PATH}/examples/*; do
+then
-    # traverse examples within each category
+    START_NUM=0
-    for example in ${category}/*; do
+    END_NUM=999
-        [ -f ${example}/Makefile ] || continue
+else
-        echo "Building ${example} as ${EXAMPLE_NUM}..."
+    JOB_NAME=$1
-        mkdir -p example_builds/${EXAMPLE_NUM}
-        cp -r ${example} example_builds/${EXAMPLE_NUM}
-        pushd example_builds/${EXAMPLE_NUM}/`basename ${example}`
 
+    # parse text prefix at the beginning of string 'some_your_text_NUM'
+    # (will be 'some_your_text' without last '_')
+    JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' )
+    [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
+
+    # parse number 'NUM' at the end of string 'some_your_text_NUM'
+    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_([0-9]+)$/\1/p' )
+    [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
+
+    # count number of the jobs
+    NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${IDF_PATH}/.gitlab-ci.yml" )
+    [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad"
+
+    # count number of examples
+    NUM_OF_EXAMPLES=$( find ${IDF_PATH}/examples/ -type f -name Makefile | wc -l )
+    [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad"
+
+    # separate intervals
+    #57 / 5 == 12
+    NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
+    [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
+
+    # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
+    START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
+    [ -z ${START_NUM} ] && die "START_NUM is bad"
+
+    END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} ))
+    [ -z ${END_NUM} ] && die "END_NUM is bad"
+fi
+
+build_example () {
+    local ID=$1
+    shift
+    local MAKE_FILE=$1
+    shift
+
+    local EXAMPLE_DIR=$(dirname "${MAKE_FILE}")
+    local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
+
+    echo "Building ${EXAMPLE_NAME} as ${ID}..."
+    mkdir -p "example_builds/${ID}"
+    cp -r "${EXAMPLE_DIR}" "example_builds/${ID}"
+    pushd "example_builds/${ID}/${EXAMPLE_NAME}"
         # be stricter in the CI build than the default IDF settings
         export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations"
         export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
 
         # build non-verbose first
-        BUILDLOG=$(mktemp -t examplebuild.XXXX.log)
+        local BUILDLOG=$(mktemp -t examplebuild.XXXX.log)
         (
-            set -o pipefail  # so result of make all isn't lost when piping to tee
-            set -e
             make clean defconfig
-            make $* all 2>&1 | tee $BUILDLOG
+            make all 2>&1 | tee "${BUILDLOG}"
-        ) || { RESULT=$?; FAILED_EXAMPLES+=" ${example}"; make V=1; } # verbose output for errors
+        ) || { RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}"; make V=1; } # verbose output for errors
-        popd
+    popd
+
+    if grep ": warning:" "${BUILDLOG}" 2>&1 >> "${LOG_WARNINGS}"; then
+        [ $RESULT -eq 0 ] && RESULT=$RESULT_WARNINGS
+        FAILED_EXAMPLES+=" ${EXAMPLE_NAME} (warnings)"
+    fi
+
+    grep -i error "${BUILDLOG}" 2>&1 >> "${LOG_WARNINGS}" || :
+
+    rm -f "${BUILDLOG}"
+}
+
+EXAMPLE_NUM=0
+
+find ${IDF_PATH}/examples/ -type f -name Makefile | \
+while read FN
+do
+    if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
+    then
         EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
+        continue
+    fi
+    echo ">>> example [ ${EXAMPLE_NUM} ] - $FN"
 
-        if grep -q ": warning:" $BUILDLOG; then
+    build_example "${EXAMPLE_NUM}" "${FN}"
-            [ $RESULT -eq 0 ] && RESULT=$RESULT_WARNINGS
-            FAILED_EXAMPLES+=" ${example} (warnings)"
-        fi
 
-        rm -f $BUILDLOG
+    EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
-    done
 done
 
+# show warnings
+echo -e "\nFound issues:"
+# pattern is: not 'error.o' and not '-Werror'
+grep -v "error.o\|\-Werror" -- "${LOG_WARNINGS}" || echo -e "\tNone"
+rm -f "${LOG_WARNINGS}"
+
 if [ $RESULT -eq $RESULT_WARNINGS ]; then
     echo "Build would have passed, except for warnings."
 fi
 
 [ $RESULT -eq 0 ] || echo "Failed examples: $FAILED_EXAMPLES"
 
-exit $RESULT
+echo -e "\nReturn code = $RESULT"
 
+exit $RESULT