unit-test-app: allow specifying subset of tests for configuration

For some test configurations, not all tests need to be run. This
change allows adding a comment in the configuration file of the
following form:

unit-test-app makefile will use it to determine which components to
build. If the comment is not present, all components will be built
(TESTS_ALL=1).

Also add test configuration for libsodium. It is a separate test
configuration due to the large binary size generated when building
libsodium tests.

.gitlab-ci.yml

@@ -161,11 +161,11 @@ build_esp_idf_tests:
     expire_in: 6 mos
   script:
     - cd tools/unit-test-app
-    - make help # make sure kconfig tools are built in single process
+    - MAKEFLAGS= make help # make sure kconfig tools are built in single process
     - make ut-clean-all-configs
     - export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations"
     - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
-    - make ut-build-all-configs TESTS_ALL=1
+    - make ut-build-all-configs
     - python tools/UnitTestParser.py
 
 .build_examples_template: &build_examples_template
@@ -863,6 +863,18 @@ UT_001_27:
     - ESP32_IDF
     - UT_T1_1
 
+UT_001_28:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
+UT_001_29:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
 UT_002_01:
   <<: *unit_test_template
   tags:

components/libsodium/test/component.mk

@@ -5,7 +5,9 @@
 LS_TESTDIR := ../libsodium/test/default
 LS_TEST_OBJDIR := libsodium/test/default
 
-ifdef TESTS_ALL
+TESTS_ALL ?= 0
+
+ifeq ($(TESTS_ALL),1)
 $(info not linking libsodium tests, use 'TEST_COMPONENTS=libsodium' to test it)
 else
 COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive

tools/unit-test-app/Makefile

@@ -5,10 +5,11 @@
 
 PROJECT_NAME := unit-test-app
 
-NON_INTERACTIVE_TARGET += ut-apply-config-% ut-clean-%
-
 include $(IDF_PATH)/make/project.mk
 
+ifeq ($(MAKELEVEL),0)
+# Define helper targets only when not recursing
+
 # List of unit-test-app configurations.
 # Each file in configs/ directory defines a configuration. The format is the
 # same as sdkconfig file. Configuration is applied on top of sdkconfig.defaults
@@ -61,12 +62,22 @@ $(BINARIES_DIR)/%/$(PROJECT_NAME).bin: configs/%
 	cat sdkconfig.defaults > $(BUILDS_DIR)/$*/sdkconfig.defaults
 	echo "" >> $(BUILDS_DIR)/$*/sdkconfig.defaults # in case there is no trailing newline in sdkconfig.defaults
 	cat configs/$* >> $(BUILDS_DIR)/$*/sdkconfig.defaults
+
 	# Build, tweaking paths to sdkconfig and sdkconfig.defaults
 	$(summary) BUILD_CONFIG $(BUILDS_DIR)/$*
-	$(MAKE) defconfig all \
+	# 'TEST_COMPONENTS=names' option can be added to configs/$* to limit the set
+	# of tests to build for given configuration.
+	# Build all tests if this option is not present.
+	test_components=`sed -n 's/^TEST_COMPONENTS=\(.*\)/\1/p' configs/$*`; \
+		tests_all=`test -n "$${test_components}"; echo $${?}`; \
+		exclude_components=`sed -n 's/^EXCLUDE_COMPONENTS=\(.*\)/\1/p' configs/$*`; \
+	$(MAKE) defconfig list-components all \
 		BUILD_DIR_BASE=$(BUILDS_DIR)/$* \
 		SDKCONFIG=$(BUILDS_DIR)/$*/sdkconfig \
-		SDKCONFIG_DEFAULTS=$(BUILDS_DIR)/$*/sdkconfig.defaults
+		SDKCONFIG_DEFAULTS=$(BUILDS_DIR)/$*/sdkconfig.defaults \
+		TEST_COMPONENTS="$${test_components}" \
+		TESTS_ALL=$${tests_all} \
+		EXCLUDE_COMPONENTS="$${exclude_components}"
 	$(MAKE) print_flash_cmd \
 		BUILD_DIR_BASE=$(BUILDS_DIR)/$* \
 		SDKCONFIG=$(BUILDS_DIR)/$*/sdkconfig \
@@ -87,17 +98,32 @@ ut-help:
 	@echo "make ut-build-NAME - Build unit-test-app with configuration provided in configs/NAME."
 	@echo "                  Build directory will be builds/NAME/, output binaries will be"
 	@echo "                  under output/NAME/"
-	@echo "make ut-clean-NAME - Remove build and output directories for configuration NAME."
 	@echo ""
 	@echo "make ut-build-all-configs - Build all configurations defined in configs/ directory."
 	@echo ""
+	@echo "Above targets determine list of components to be built from configs/NAME files."
+	@echo "To build custom subset of components use 'make ut-apply-config-NAME' and then 'make all'."
+	@echo ""
 	@echo "make ut-apply-config-NAME - Generates configuration based on configs/NAME in sdkconfig"
 	@echo "                         file. After this, normal all/flash targets can be used."
 	@echo "                         Useful for development/debugging."
 	@echo ""
+	@echo "make ut-clean-NAME - Remove build and output directories for configuration NAME."
+	@echo ""
 
 help: ut-help
 
 .PHONY: ut-build-all-configs ut-clean-all-configs \
 		$(CONFIG_BUILD_TARGETS) $(CONFIG_CLEAN_TARGETS) $(CONFIG_APPLY_TARGETS) \
 		ut-help
+
+NON_INTERACTIVE_TARGET += ut-apply-config-% ut-clean-% ut-build-% \
+						  ut-build-all-configs ut-clean-all-configs
+
+else # MAKELEVEL == 0
+
+# If recursing, print the actual list of tests being built
+
+$(info TESTS $(foreach comp,$(TEST_COMPONENT_NAMES),$(patsubst %_test,%,$(comp))))
+
+endif # MAKELEVEL == 0

tools/unit-test-app/configs/default

@@ -1 +1 @@
-# default config — nothing to set here
+EXCLUDE_COMPONENTS=libsodium

tools/unit-test-app/configs/libsodium

@@ -0,0 +1 @@
+TEST_COMPONENTS=libsodium

tools/unit-test-app/configs/psram

@@ -1 +1,2 @@
+EXCLUDE_COMPONENTS=libsodium
 CONFIG_SPIRAM_SUPPORT=y

tools/unit-test-app/configs/single_core

@@ -1,2 +1,3 @@
+EXCLUDE_COMPONENTS=libsodium
 CONFIG_MEMMAP_SMP=n
 CONFIG_FREERTOS_UNICORE=y