From 2544355301782c79f9a7f256c6987178fa80411e Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 1 Sep 2017 13:42:39 +1000 Subject: [PATCH 1/6] build_examples: Small cleanups * Override MAKEFLAGS via make not via shell variables * Remove build_examples in gitlab-ci * Don't use mktemp for any logs --- .gitlab-ci.yml | 1 + tools/ci/build_examples.sh | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d22f55af2..56664145b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,6 +142,7 @@ build_esp_idf_tests: script: # it's not possible to build 100% out-of-tree and have the "artifacts" # mechanism work, but this is the next best thing + - rm -rf build_examples - mkdir build_examples - cd build_examples # build some of examples diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index 8cdb05ae5..368c1efa8 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -44,6 +44,8 @@ die() { [ -z ${IDF_PATH} ] && die "IDF_PATH is not set" +echo "build_examples running in ${PWD}" + # only 0 or 1 arguments [ $# -le 1 ] || die "Have to run as $(basename $0) []" @@ -55,7 +57,7 @@ RESULT=0 FAILED_EXAMPLES="" RESULT_WARNINGS=22 # magic number result code for "warnings found" -LOG_WARNINGS=$(mktemp -t example_all.XXXX.log) +LOG_WARNINGS=${PWD}/build_warnings.log if [ $# -eq 0 ] then @@ -114,7 +116,7 @@ build_example () { # build non-verbose first local BUILDLOG=${PWD}/examplebuild.${ID}.log ( - MAKEFLAGS= make clean defconfig &> >(tee -a "${BUILDLOG}") && + make MAKEFLAGS= clean defconfig &> >(tee -a "${BUILDLOG}") && make all &> >(tee -a "${BUILDLOG}") ) || { RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" From c275ad4ca486f7599b4bb94420c7fa336fd7bded Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 1 Sep 2017 17:28:13 +1000 Subject: [PATCH 2/6] build system: Add dependencies to component clean steps Build directory must exist, component_project_vars.mk must be generated if it is going to be. --- components/bootloader/Makefile.projbuild | 2 +- make/project.mk | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/bootloader/Makefile.projbuild b/components/bootloader/Makefile.projbuild index a8c10067b..7a35dc8ad 100644 --- a/components/bootloader/Makefile.projbuild +++ b/components/bootloader/Makefile.projbuild @@ -118,6 +118,6 @@ ifndef CONFIG_SECURE_BOOT_ENABLED all_binaries: $(BOOTLOADER_BIN) endif -bootloader-clean: +bootloader-clean: $(SDKCONFIG_MAKEFILE) $(BOOTLOADER_MAKE) app-clean rm -f $(SECURE_BOOTLOADER_KEY) $(BOOTLOADER_DIGEST_BIN) diff --git a/make/project.mk b/make/project.mk index d2f3cfcd5..195147bd8 100644 --- a/make/project.mk +++ b/make/project.mk @@ -361,10 +361,10 @@ endef define GenerateComponentTargets .PHONY: component-$(2)-build component-$(2)-clean -component-$(2)-build: check-submodules +component-$(2)-build: check-submodules $(call prereq_if_explicit, component-$(2)-clean) | $(BUILD_DIR_BASE)/$(2) $(call ComponentMake,$(1),$(2)) build -component-$(2)-clean: +component-$(2)-clean: | $(BUILD_DIR_BASE)/$(2) $(BUILD_DIR_BASE)/$(2)/component_project_vars.mk $(call ComponentMake,$(1),$(2)) clean $(BUILD_DIR_BASE)/$(2): @@ -406,7 +406,7 @@ size-components: $(APP_ELF) # NB: this ordering is deliberate (app-clean before config-clean), # so config remains valid during all component clean targets -config-clean: app-clean +config-clean: app-clean $(call prereq_if_explicit,bootloader-clean) clean: config-clean # phony target to check if any git submodule listed in COMPONENT_SUBMODULES are missing From 774c9d0a61f4059376bc423f5ece8951481997e3 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 1 Sep 2017 17:29:51 +1000 Subject: [PATCH 3/6] ci: Add two more example build jobs per CI pass --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56664145b..f0ed5734e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -163,6 +163,12 @@ build_examples_03: build_examples_04: <<: *build_examples_template +build_examples_05: + <<: *build_examples_template + +build_examples_06: + <<: *build_examples_template + build_docs: stage: build image: $CI_DOCKER_REGISTRY/esp32-ci-env From 97efaab27b067edf0a62c0cd023c94b8df294341 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 4 Sep 2017 11:15:04 +1000 Subject: [PATCH 4/6] build system tests: Run inside PWD same as build_examples.sh --- .gitlab-ci.yml | 7 +++++-- tools/ci/build_examples.sh | 2 +- tools/ci/test_build_system.sh | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0ed5734e..9e68170ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -235,8 +235,11 @@ test_build_system: - build_test dependencies: [] script: - - ./tools/ci/test_configure_ci_environment.sh - - ./tools/ci/test_build_system.sh + - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh + - rm -rf test_build_system + - mkdir test_build_system + - cd test_build_system + - ${IDF_PATH}/tools/ci/test_build_system.sh test_report: stage: test_report diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index 368c1efa8..ec7ff16eb 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -5,7 +5,7 @@ # # Runs as part of CI process. # -# Assumes CWD is an out-of-tree build directory, and will copy examples +# Assumes PWD is an out-of-tree build directory, and will copy examples # to individual subdirectories, one by one. # # diff --git a/tools/ci/test_build_system.sh b/tools/ci/test_build_system.sh index 0f2472422..e7a3c7316 100755 --- a/tools/ci/test_build_system.sh +++ b/tools/ci/test_build_system.sh @@ -3,11 +3,12 @@ # Test the build system for basic consistency # # A bash script that tests some likely make failure scenarios in a row -# Creates its own test build directory under TMP and cleans it up when done. +# +# Assumes PWD is an out-of-tree build directory, and will create a +# subdirectory inside it to run build tests in. # # Environment variables: # IDF_PATH - must be set -# TMP - can override /tmp location for build directory # ESP_IDF_TEMPLATE_GIT - Can override git clone source for template app. Otherwise github. # NOCLEANUP - Set to '1' if you want the script to leave its temporary directory when done, for post-mortem. # @@ -26,7 +27,6 @@ # Set up some variables # -[ -z ${TMP} ] && TMP="/tmp" # override ESP_IDF_TEMPLATE_GIT to point to a local dir if you're testing and want fast iterations [ -z ${ESP_IDF_TEMPLATE_GIT} ] && ESP_IDF_TEMPLATE_GIT=https://github.com/espressif/esp-idf-template.git @@ -205,7 +205,7 @@ function failure() FAILURES="${FAILURES}${STATUS} :: $1\n" } -TESTDIR=${TMP}/build_system_tests_$$ +TESTDIR=${PWD}/build_system_tests_$$ mkdir -p ${TESTDIR} # set NOCLEANUP=1 if you want to keep the test directory around # for post-mortem debugging From 867b20837f8107689996e7e278878e012acc23cc Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 4 Sep 2017 13:17:32 +1000 Subject: [PATCH 5/6] build system: Explicitly disallow 'clean' along with non-cleaning targets Too hard to stage the dependencies so that all clean steps complete before any build steps begin. Also, using and then deleting and then regenerating SDKCONFIG_MAKEFILE in one pass is really hard to manage successfully. --- make/project.mk | 19 ++++++++++++++----- tools/ci/build_examples.sh | 7 ++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/make/project.mk b/make/project.mk index 195147bd8..d22a1b1dd 100644 --- a/make/project.mk +++ b/make/project.mk @@ -45,10 +45,18 @@ help: # dependency checks ifndef MAKE_RESTARTS ifeq ("$(filter 4.% 3.81 3.82,$(MAKE_VERSION))","") -$(warning "esp-idf build system only supports GNU Make versions 3.81 or newer. You may see unexpected results with other Makes.") +$(warning esp-idf build system only supports GNU Make versions 3.81 or newer. You may see unexpected results with other Makes.) endif endif +# can't run 'clean' along with any non-clean targets +ifneq ("$(filter clean% %clean,$(MAKECMDGOALS))" ,"") +ifneq ("$(filter-out clean% %clean,$(MAKECMDGOALS))", "") +$(error esp-idf build system doesn't support running 'clean' targets along with any others. Run 'make clean' and then run other targets separately.) +endif +endif + + # make IDF_PATH a "real" absolute path # * works around the case where a shell character is embedded in the environment variable value. # * changes Windows-style C:/blah/ paths to MSYS/Cygwin style /c/blah @@ -404,10 +412,11 @@ size-files: $(APP_ELF) size-components: $(APP_ELF) $(PYTHON) $(IDF_PATH)/tools/idf_size.py --archives $(APP_MAP) -# NB: this ordering is deliberate (app-clean before config-clean), -# so config remains valid during all component clean targets -config-clean: app-clean $(call prereq_if_explicit,bootloader-clean) -clean: config-clean +# NB: this ordering is deliberate (app-clean & bootloader-clean before +# _config-clean), so config remains valid during all component clean +# targets +config-clean: app-clean bootloader-clean +clean: app-clean bootloader-clean config-clean # phony target to check if any git submodule listed in COMPONENT_SUBMODULES are missing # or out of date, and exit if so. Components can add paths to this variable. diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index ec7ff16eb..c22aaf4bd 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -116,9 +116,10 @@ build_example () { # build non-verbose first local BUILDLOG=${PWD}/examplebuild.${ID}.log ( - make MAKEFLAGS= clean defconfig &> >(tee -a "${BUILDLOG}") && - make all &> >(tee -a "${BUILDLOG}") - ) || { + make MAKEFLAGS= clean && + make MAKEFLAGS= defconfig && + make all + ) &> >(tee -a "${BUILDLOG}") || { RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" make MAKEFLAGS= V=1 clean defconfig && make V=1 # verbose output for errors } From 2806b57fc17688b3c24f726d324d2f3fa6f48f2a Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 4 Sep 2017 14:50:52 +1000 Subject: [PATCH 6/6] build system: Don't make menuconfig if "make clean" run with no sdkconfig --- make/project_config.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/make/project_config.mk b/make/project_config.mk index c0369f95f..c92d5292e 100644 --- a/make/project_config.mk +++ b/make/project_config.mk @@ -21,9 +21,9 @@ $(KCONFIG_TOOL_DIR)/mconf $(KCONFIG_TOOL_DIR)/conf: $(MAKE) -C $(KCONFIG_TOOL_DIR) ifeq ("$(wildcard $(SDKCONFIG))","") -ifeq ("$(filter defconfig, $(MAKECMDGOALS))","") -# if no configuration file is present and defconfig is not a named -# target, run defconfig then menuconfig to get the initial config +ifeq ("$(filter defconfig clean% %clean, $(MAKECMDGOALS))","") +# if no configuration file is present and defconfig or clean +# is not a named target, run defconfig then menuconfig to get the initial config $(SDKCONFIG): menuconfig menuconfig: defconfig else