From 058d13c3513fc4b64b3c21580696410844966c27 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 30 Oct 2019 18:36:51 +0100 Subject: [PATCH 1/4] ci: fixes for CMake build system test on macOS --- tools/ci/test_build_system_cmake.sh | 64 +++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/tools/ci/test_build_system_cmake.sh b/tools/ci/test_build_system_cmake.sh index 19a36d2d4..bd9eaa4e4 100755 --- a/tools/ci/test_build_system_cmake.sh +++ b/tools/ci/test_build_system_cmake.sh @@ -94,7 +94,7 @@ function run_tests() print_status "Partial build doesn't compile anything by default" take_build_snapshot # verify no build files are refreshed by a partial make - ALL_BUILD_FILES=$(find ${BUILD} -type f | sed "s@${BUILD}/@@" | grep -v '^.') + ALL_BUILD_FILES=$(find ${BUILD} -type f | ${SED} "s@${BUILD}/@@" | grep -v '^.') idf.py build || failure "Partial build failed" assert_not_rebuilt ${ALL_BUILD_FILES} @@ -127,7 +127,7 @@ function run_tests() print_status "Can set COMPONENT_SRCS with spaces" clean_build_dir touch main/main2.c - sed -i 's/^set(COMPONENT_SRCS.*/set(COMPONENT_SRCS "main.c main2.c")/' main/CMakeLists.txt + ${SED} -i 's/^set(COMPONENT_SRCS.*/set(COMPONENT_SRCS "main.c main2.c")/' main/CMakeLists.txt idf.py build || failure "Set COMPONENT_SRCS with spaces build failed" git checkout -- main/CMakeLists.txt rm main/main2.c @@ -136,7 +136,7 @@ function run_tests() clean_build_dir OUTOFTREE_BUILD=${TESTDIR}/alt_build idf.py -B "${OUTOFTREE_BUILD}" build || failure "Failed to build with out-of-tree build dir" - NEW_BUILD_FILES=$(find ${OUTOFREE_BUILD} -type f) + NEW_BUILD_FILES=$(find ${OUTOFTREE_BUILD} -type f) if [ -z "${NEW_BUILD_FILES}" ]; then failure "No files found in new build directory!" fi @@ -159,7 +159,9 @@ function run_tests() # make a copy of esp-idf and CRLFify it CRLF_ESPIDF=${TESTDIR}/esp-idf-crlf mkdir -p ${CRLF_ESPIDF} - cp -r ${IDF_PATH}/* ${CRLF_ESPIDF} + # Note: trailing slash after ${IDF_PATH} avoids creating esp-idf directory inside ${CRLF_ESPIDF} + rsync -a --exclude ${TESTDIR} ${IDF_PATH}/ ${CRLF_ESPIDF} + # cp -r ${IDF_PATH}/* ${CRLF_ESPIDF} # don't CRLFify executable files, as Linux will fail to execute them find ${CRLF_ESPIDF} -name .git -prune -name build -prune -type f ! -perm 755 -exec unix2dos {} \; IDF_PATH=${CRLF_ESPIDF} idf.py build || failure "Failed to build with CRLFs in source" @@ -203,7 +205,7 @@ function run_tests() idf.py build take_build_snapshot # need to actually change config, or cmake is too smart to rebuild - sed -i.bak s/^\#\ CONFIG_FREERTOS_UNICORE\ is\ not\ set/CONFIG_FREERTOS_UNICORE=y/ sdkconfig + ${SED} -i.bak s/^\#\ CONFIG_FREERTOS_UNICORE\ is\ not\ set/CONFIG_FREERTOS_UNICORE=y/ sdkconfig idf.py build # check the sdkconfig.h file was rebuilt assert_rebuilt config/sdkconfig.h @@ -220,7 +222,7 @@ function run_tests() take_build_snapshot # Need to actually change the build config, or CMake won't do anything cp CMakeLists.txt CMakeLists.bak - sed -i.bak 's/^project(/add_compile_options("-DUSELESS_MACRO_DOES_NOTHING=1")\nproject\(/' CMakeLists.txt + ${SED} -i.bak 's/^project(/add_compile_options("-DUSELESS_MACRO_DOES_NOTHING=1")\nproject\(/' CMakeLists.txt idf.py build || failure "Build failed" mv CMakeLists.bak CMakeLists.txt # similar to previous test @@ -242,7 +244,7 @@ function run_tests() print_status "Can build with IDF_PATH set via cmake cache not environment" clean_build_dir - sed -i.bak 's/ENV{IDF_PATH}/{IDF_PATH}/' CMakeLists.txt + ${SED} -i.bak 's/ENV{IDF_PATH}/{IDF_PATH}/' CMakeLists.txt export IDF_PATH_BACKUP="$IDF_PATH" (unset IDF_PATH && cd build && @@ -253,7 +255,7 @@ function run_tests() print_status "Can build with IDF_PATH unset and inferred by build system" clean_build_dir - sed -i.bak "s%\$ENV{IDF_PATH}%\${ci_idf_path}%" CMakeLists.txt # expand to a hardcoded path + ${SED} -i.bak "s%\$ENV{IDF_PATH}%\${ci_idf_path}%" CMakeLists.txt # expand to a hardcoded path (ci_idf_path=${IDF_PATH} && unset IDF_PATH && cd build && cmake -G Ninja -D ci_idf_path=${ci_idf_path} .. && ninja) || failure "Ninja build failed" mv CMakeLists.txt.bak CMakeLists.txt @@ -261,7 +263,7 @@ function run_tests() print_status "Can build with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set" clean_build_dir - sed -i.bak 's/ENV{IDF_PATH}/{IDF_PATH}/' CMakeLists.txt + ${SED} -i.bak 's/ENV{IDF_PATH}/{IDF_PATH}/' CMakeLists.txt export IDF_PATH_BACKUP="$IDF_PATH" mv main/Kconfig.projbuild main/Kconfig.projbuild_bak echo "source \"\$IDF_PATH/examples/wifi/getting_started/station/main/Kconfig.projbuild\"" > main/Kconfig.projbuild @@ -280,7 +282,7 @@ function run_tests() mkdir -p ${IDF_PATH}/components/xtensa/$fake_target/include touch components/$fake_target/CMakeLists.txt cp ${IDF_PATH}/tools/cmake/toolchain-esp32.cmake components/$fake_target/toolchain-$fake_target.cmake - sed -i.bak '/cmake_minimum_required/ a\ + ${SED} -i.bak '/cmake_minimum_required/ a\ set(COMPONENTS esptool_py)' CMakeLists.txt print_status "Can override IDF_TARGET from environment" @@ -339,7 +341,7 @@ function run_tests() mkdir -p main/main/main # move main component contents to another directory mv main/* main/main/main cp CMakeLists.txt CMakeLists.bak # set EXTRA_COMPONENT_DIRS to point to the other directory - sed -i "s%cmake_minimum_required(VERSION \([0-9]\+\).\([0-9]\+\))%cmake_minimum_required(VERSION \1.\2)\nset(EXTRA_COMPONENT_DIRS main/main/main)%" CMakeLists.txt + ${SED} -i "s%cmake_minimum_required(VERSION \([0-9]\+\).\([0-9]\+\))%cmake_minimum_required(VERSION \1.\2)\nset(EXTRA_COMPONENT_DIRS main/main/main)%" CMakeLists.txt idf.py build || failure "Build with EXTRA_COMPONENT_DIRS set failed" mv CMakeLists.bak CMakeLists.txt # revert previous modifications mv main/main/main/* main @@ -548,7 +550,7 @@ endmenu\n" >> ${IDF_PATH}/Kconfig; clean_build_dir cp CMakeLists.txt CMakeLists.txt.bak printf "\nidf_component_get_property(srcs main SRCS)\nmessage(STATUS SRCS:\${srcs})" >> CMakeLists.txt - (idf.py reconfigure | grep "SRCS:$(realpath main/main.c)") || failure "Component properties should be set" + (idf.py reconfigure | grep "SRCS:$(${REALPATH} main/main.c)") || failure "Component properties should be set" rm -rf CMakeLists.txt mv CMakeLists.txt.bak CMakeLists.txt rm -rf CMakeLists.txt.bak @@ -598,12 +600,25 @@ mkdir -p ${TESTDIR} SNAPSHOT=${TESTDIR}/snapshot BUILD=${TESTDIR}/template/build +IS_DARWIN= +export SED=sed +export REALPATH=realpath +if [ "$(uname -s)" = "Darwin" ]; then + IS_DARWIN=1 + export SED=gsed + export REALPATH=grealpath +fi # copy all the build output to a snapshot directory function take_build_snapshot() { rm -rf ${SNAPSHOT} cp -ap ${TESTDIR}/template/build ${SNAPSHOT} + if [ -n "$IS_DARWIN" ]; then + # wait at least 1 second before the next build, for the test in + # file_was_rebuilt to work + sleep 1 + fi } # verify that all the arguments are present in the build output directory @@ -620,12 +635,21 @@ function assert_built() # Test if a file has been rebuilt. function file_was_rebuilt() { - # can't use [ a -ot b ] here as -ot only gives second resolution - # but stat -c %y seems to be microsecond at least for tmpfs, ext4.. - if [ "$(stat -c %y ${SNAPSHOT}/$1)" != "$(stat -c %y ${BUILD}/$1)" ]; then - return 0 + if [ -z "$IS_DARWIN" ]; then + # can't use [ a -ot b ] here as -ot only gives second resolution + # but stat -c %y seems to be microsecond at least for tmpfs, ext4.. + if [ "$(stat -c %y ${SNAPSHOT}/$1)" != "$(stat -c %y ${BUILD}/$1)" ]; then + return 0 + else + return 1 + fi else - return 1 + # macOS: work around 1-second resolution by adding a sleep in take_build_snapshot + if [ ${SNAPSHOT}/$1 -ot ${BUILD}/$1 ]; then + return 0 + else + return 1 + fi fi } @@ -663,7 +687,11 @@ function assert_not_rebuilt() # do a "clean" that doesn't depend on idf.py function clean_build_dir() { - rm -rf --preserve-root ${BUILD}/* ${BUILD}/.* + PRESERVE_ROOT_ARG= + if [ -z "$IS_DARWIN" ]; then + PRESERVE_ROOT_ARG=--preserve-root + fi + rm -rf $PRESERVE_ROOT_ARG ${BUILD}/* ${BUILD}/.* } cd ${TESTDIR} From cfe9244a96daec2530709f0e22294db26dea4ce5 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 30 Oct 2019 15:00:18 +0100 Subject: [PATCH 2/4] ci: add build system test on macOS --- .gitlab-ci.yml | 11 +++++++++++ tools/ci/config/build.yml | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58941f240..f3e8f7c11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -135,6 +135,17 @@ after_script: dependencies: [] extends: .before_script_lesser_nofilter +.macos_build_template: + stage: build + tags: + - macos_shell + dependencies: [] + before_script: + - *apply_bot_filter + - *setup_tools_unless_target_test + - *setup_custom_toolchain + - source tools/ci/configure_ci_environment.sh + include: - '/tools/ci/config/build.yml' - '/tools/ci/config/assign-test.yml' diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 6fb4c5e4a..b511dd291 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -308,6 +308,27 @@ test_build_system_cmake: - cd test_build_system - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh +test_build_system_cmake_macos: + extends: .macos_build_template + only: + refs: + - master + - /^release\/v/ + - /^v\d+\.\d+(\.\d+)?($|-)/ + - schedules + - triggers + - pipelines + - web + variables: + - $CI_PIPELINE_SOURCE != "push" && $BOT_LABEL_MACOS_TEST != null + - $CI_PIPELINE_SOURCE == "push" + script: + - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh + - rm -rf test_build_system + - mkdir test_build_system + - cd test_build_system + - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh + build_docker: stage: build image: espressif/docker-builder:1 From f61fd8a664e88e2fa712320f1a898ea2a4ba8bee Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 30 Oct 2019 15:29:09 +0100 Subject: [PATCH 3/4] tools: remove spaces from toolchain URLs --- tools/tools.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tools.json b/tools/tools.json index 6c0ff0627..ed7882f2e 100644 --- a/tools/tools.json +++ b/tools/tools.json @@ -95,14 +95,14 @@ "macos": { "sha256": "e566b9f1288ef3bd89b08e9807e88c37f760d48e74b84eee9e9642bb5c0a0bbd", "size": 69793197, - "url": " https://dl.espressif.com/dl/toolchains/preview/xtensa-esp32s2-elf-gcc8_2_0-esp32s2-dev-4-g3a626e-macos.tar.gz " + "url": "https://dl.espressif.com/dl/toolchains/preview/xtensa-esp32s2-elf-gcc8_2_0-esp32s2-dev-4-g3a626e-macos.tar.gz" }, "name": "esp32s2-dev-4-g3a626e9-8.2.0", "status": "recommended", "win32": { "sha256": "6ebdcd01de2c8c50ac38446dd4421bb44d320b30b85727c49db26a703ac2c381", "size": 73352955, - "url": " https://dl.espressif.com/dl/toolchains/preview/xtensa-esp32s2-elf-gcc8_2_0-esp32s2-dev-4-g3a626e-win32.zip" + "url": "https://dl.espressif.com/dl/toolchains/preview/xtensa-esp32s2-elf-gcc8_2_0-esp32s2-dev-4-g3a626e-win32.zip" }, "win64": { "sha256": "6ebdcd01de2c8c50ac38446dd4421bb44d320b30b85727c49db26a703ac2c381", From 9279b72a6cd04ef0cf1ac4d1cfaf416966ded7e3 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 30 Oct 2019 16:19:17 +0100 Subject: [PATCH 4/4] idf_tools.py: fix virtualenv issue for macOS with homebrew --- tools/idf_tools.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/idf_tools.py b/tools/idf_tools.py index a04cc1c9c..e9978ec60 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -1315,6 +1315,11 @@ def main(argv): global global_idf_tools_path global_idf_tools_path = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(IDF_TOOLS_PATH_DEFAULT) + # On macOS, unset __PYVENV_LAUNCHER__ variable if it is set. + # Otherwise sys.executable keeps pointing to the system Python, even when a python binary from a virtualenv is invoked. + # See https://bugs.python.org/issue22490#msg283859. + os.environ.pop('__PYVENV_LAUNCER__', None) + if sys.version_info.major == 2: try: global_idf_tools_path.decode('ascii')