Merge branch 'master' into feature/newlib_locking

.gitlab-ci.yml

@@ -25,6 +25,14 @@ test_nvs_on_host:
     - cd components/nvs_flash/test
     - make test
 
+test_build_system:
+  stage: test
+  image: espressif/esp32-ci-env
+  variables:
+    IDF_PATH: "$CI_PROJECT_DIR"
+  script:
+    - ./make/build_system_tests.sh
+
 push_master_to_github:
   stage: deploy
   only:

README.buildenv

@@ -239,7 +239,7 @@ COMPONENT_EXTRA_CLEAN := test_tjpgd_logo.h
 graphics_lib.o: logo.h
 
 logo.h: $(COMPONENT_PATH)/logo.bmp
-	$(vecho) BMP2H $@
+	$(summary) BMP2H $@
 	$(Q) bmp2h -i $^ -o $@
 
 include $(IDF_PATH)/make/component.mk

components/bootloader/Makefile.projbuild

@@ -13,7 +13,7 @@ BOOTLOADER_COMPONENT_PATH := $(COMPONENT_PATH)
 BOOTLOADER_BUILD_DIR=$(BUILD_DIR_BASE)/bootloader
 BOOTLOADER_BIN=$(BOOTLOADER_BUILD_DIR)/bootloader.bin
 
-.PHONY: bootloader-clean bootloader-flash bootloader
+.PHONY: bootloader-clean bootloader-flash bootloader $(BOOTLOADER_BIN)
 
 $(BOOTLOADER_BIN): $(COMPONENT_PATH)/src/sdkconfig
 	$(Q) PROJECT_PATH= \

components/freertos/FreeRTOS-openocd.c

@@ -0,0 +1,23 @@
+/*
+ * Since at least FreeRTOS V7.5.3 uxTopUsedPriority is no longer
+ * present in the kernel, so it has to be supplied by other means for
+ * OpenOCD's threads awareness.
+ *
+ * Add this file to your project, and, if you're using --gc-sections,
+ * ``--undefined=uxTopUsedPriority'' (or
+ * ``-Wl,--undefined=uxTopUsedPriority'' when using gcc for final
+ * linking) to your LDFLAGS; same with all the other symbols you need.
+ */
+
+#include "FreeRTOS.h"
+#include "sdkconfig.h"
+
+#ifdef __GNUC__
+#define USED __attribute__((used))
+#else
+#define USED
+#endif
+
+#ifdef CONFIG_FREERTOS_DEBUG_OCDAWARE
+const int USED uxTopUsedPriority = configMAX_PRIORITIES - 1;
+#endif

components/freertos/Kconfig

@@ -147,4 +147,12 @@ config FREERTOS_ASSERT_DISABLE
 
 endchoice
 
+config FREERTOS_BREAK_ON_SCHEDULER_START_JTAG
+	bool "Stop program on scheduler start when JTAG/OCD is detected"
+	depends on FREERTOS_DEBUG_OCDAWARE
+	default y
+	help
+		If JTAG/OCD is connected, stop execution when the scheduler is started and the first
+		task is executed.
+
 endmenu

components/freertos/Makefile

@@ -2,6 +2,7 @@
 # Component Makefile
 #
 
+COMPONENT_ADD_LDFLAGS = -l$(COMPONENT_NAME) -Wl,--undefined=uxTopUsedPriority
 COMPONENT_ADD_INCLUDEDIRS := include
 COMPONENT_PRIV_INCLUDEDIRS := include/freertos
 

components/freertos/include/freertos/panic.h

@@ -0,0 +1,7 @@
+#ifndef PANIC_H
+#define PANIC_H
+
+void setBreakpointIfJtag(void *fn);
+
+
+#endif

components/freertos/include/freertos/portmacro.h

@@ -118,8 +118,11 @@ typedef unsigned portBASE_TYPE	UBaseType_t;
 // portbenchmark
 #include "portbenchmark.h"
 
+#include "sdkconfig.h"
 
 #define portMUX_DEBUG
+#define portFIRST_TASK_HOOK CONFIG_FREERTOS_BREAK_ON_SCHEDULER_START_JTAG
+
 
 typedef struct {
 	volatile uint32_t mux;

components/freertos/panic.c

@@ -142,6 +142,17 @@ void panicHandler(XtExcFrame *frame) {
 	commonErrorHandler(frame);
 }
 
+static void setFirstBreakpoint(uint32_t pc) {
+	asm(
+		"wsr.ibreaka0 %0\n" \
+		"rsr.ibreakenable a3\n" \
+		"movi a4,1\n" \
+		"or a4, a4, a3\n" \
+		"wsr.ibreakenable a4\n" \
+		::"r"(pc):"a3","a4");
+	return;
+}
+
 void xt_unhandled_exception(XtExcFrame *frame) {
 	int *regs=(int*)frame;
 	int x;
@@ -158,14 +169,7 @@ void xt_unhandled_exception(XtExcFrame *frame) {
 		panicPutStr(". Setting bp and returning..\r\n");
 		//Stick a hardware breakpoint on the address the handler returns to. This way, the OCD debugger
 		//will kick in exactly at the context the error happened.
-		asm(
+		setFirstBreakpoint(regs[1]);
-			"wsr.ibreaka0 %0\n" \
-			"rsr.ibreakenable a3\n" \
-			"movi a4,1\n" \
-			"or a4, a4, a3\n" \
-			"wsr.ibreakenable a4\n" \
-			::"r"(regs[1]):"a3","a4");
-		return;
 	}
 	panicPutStr(". Exception was unhandled.\r\n");
 	commonErrorHandler(frame);
@@ -209,3 +213,9 @@ void commonErrorHandler(XtExcFrame *frame) {
 	while(1);
 #endif
 }
+
+
+void setBreakpointIfJtag(void *fn) {
+	if (!inOCDMode()) return;
+	setFirstBreakpoint((uint32_t)fn);
+}

components/freertos/port.c

@@ -101,6 +101,8 @@
 #include "FreeRTOS.h"
 #include "task.h"
 
+#include "panic.h"
+
 /* Defined in portasm.h */
 extern void _frxt_tick_timer_init(void);
 
@@ -385,6 +387,10 @@ portBASE_TYPE vPortCPUReleaseMutex(portMUX_TYPE *mux) {
 	return ret;
 }
 
-
+#if CONFIG_FREERTOS_BREAK_ON_SCHEDULER_START_JTAG
+void vPortFirstTaskHook(TaskFunction_t function) {
+	setBreakpointIfJtag(function);
+}
+#endif
 
 

components/freertos/tasks.c

@@ -445,6 +445,11 @@ to its original value when it is released. */
 	extern void vApplicationTickHook( void );
 #endif
 
+#if  portFIRST_TASK_HOOK
+	extern void vPortFirstTaskHook(TaskFunction_t taskfn);
+#endif
+
+
 /* File private functions. --------------------------------*/
 
 /*
@@ -707,6 +712,12 @@ BaseType_t i;
 						/* Schedule if nothing is scheduled yet, or overwrite a task of lower prio. */
 						if ( pxCurrentTCB[i] == NULL || pxCurrentTCB[i]->uxPriority <= uxPriority )
 						{
+#if portFIRST_TASK_HOOK
+							if ( i == 0) {
+								vPortFirstTaskHook(pxTaskCode);
+							}
+#endif /* configFIRST_TASK_HOOK */
+
 							pxCurrentTCB[i] = pxNewTCB;
 							break;
 						}

make/build_system_tests.sh

@@ -0,0 +1,163 @@
+#!/bin/bash
+#
+# Build system tests
+#
+# Just a bash script that tests some likely make failure scenarios in a row
+# Creates its own test build directory under TMP and cleans it up when done.
+#
+# Environment variables:
+# IDF_PATH - must be set
+# TMP - can override /tmp location for build directory
+# ESP_IDF_TEMPLATE_GIT - Can override git clone source for template app. Otherwise github.
+# NOCLEANUP - Set to '1' if you want the script to leave its temporary directory when done, for post-mortem.
+#
+
+# Set up some variables
+#
+[ -z ${TMP} ] && TMP="/tmp"
+# override ESP_IDF_TEMPLATE_GIT to point to a local dir if you're testing and want fast iterations
+[ -z ${ESP_IDF_TEMPLATE_GIT} ] && ESP_IDF_TEMPLATE_GIT=https://github.com/espressif/esp-idf-template.git
+export V=1
+
+function run_tests()
+{
+	FAILURES=
+	STATUS="Starting"
+	print_status "Checking prerequisites"
+	[ -z ${IDF_PATH} ] && echo "IDF_PATH is not set. Need path to esp-idf installation." && exit 2
+
+	print_status "Cloning template..."
+	git clone ${ESP_IDF_TEMPLATE_GIT} template
+	cd template
+
+	BOOTLOADER_BINS="bootloader/bootloader.elf bootloader/bootloader.bin"
+	APP_BINS="app-template.elf app-template.bin"
+
+	print_status "Initial clean build"
+	# if make fails here, everything fails
+	make || exit $?
+	# check all the expected build artifacts from the clean build
+	assert_built ${APP_BINS} ${BOOTLOADER_BINS} partitions_singleapp.bin
+	[ -f ${BUILD}/partition*.bin ] || failure "A partition table should have been built"
+
+	print_status "Updating component source file rebuilds component"
+	# touch a file & do a build
+	take_build_snapshot
+	touch ${IDF_PATH}/components/esp32/syscalls.c
+	make || failure "Failed to partial build"
+	assert_rebuilt ${APP_BINS} esp32/libesp32.a esp32/syscalls.o
+	assert_not_rebuilt lwip/liblwip.a freertos/libfreertos.a ${BOOTLOADER_BINS} partitions_singleapp.bin
+
+	print_status "Bootloader source file rebuilds bootloader"
+	take_build_snapshot
+	touch ${IDF_PATH}/components/bootloader/src/main/bootloader_start.c
+	make bootloader || failure "Failed to partial build bootloader"
+	assert_rebuilt ${BOOTLOADER_BINS} bootloader/main/bootloader_start.o
+	assert_not_rebuilt ${APP_BINS} partitions_singleapp.bin
+
+	print_status "Partition CSV file rebuilds partitions"
+	take_build_snapshot
+	touch ${IDF_PATH}/components/partition_table/partitions_singleapp.csv
+	make partition_table
+	assert_rebuilt partitions_singleapp.bin
+	assert_not_rebuilt app-template.bin app-template.elf ${BOOTLOADER_BINS}
+
+	print_status "Partial build doesn't compile anything by default"
+	take_build_snapshot
+	# verify no build files are refreshed by a partial make
+	ALL_BUILD_FILES=$(find ${BUILD} -type f | sed "s@${BUILD}/@@")
+	make
+	assert_not_rebuilt ${ALL_BUILD_FILES}
+
+	print_status "Cleaning should remove all files from build"
+	make clean
+	ALL_BUILD_FILES=$(find ${BUILD} -type f)
+	if [ -n "${ALL_BUILD_FILES}" ]; then
+		 failure "Files weren't cleaned: ${ALL_BUILD_FILES}"
+	fi
+
+	print_status "All tests completed"
+	if [ -n "${FAILURES}" ]; then
+		echo "Some failures were detected:"
+		echo -e "${FAILURES}"
+		exit 1
+	else
+		echo "Build tests passed."
+	fi
+}
+
+function print_status()
+{
+	echo "******** $1"
+	STATUS="$1"
+}
+
+function failure()
+{
+	echo "!!!!!!!!!!!!!!!!!!!"
+	echo "FAILURE: $1"
+	echo "!!!!!!!!!!!!!!!!!!!"
+	FAILURES="${FAILURES}${STATUS} :: $1\n"
+}
+
+TESTDIR=${TMP}/build_system_tests_$$
+mkdir -p ${TESTDIR}
+# set NOCLEANUP=1 if you want to keep the test directory around
+# for post-mortem debugging
+[ -z ${NOCLEANUP} ] && trap "rm -rf ${TESTDIR}" EXIT KILL
+
+SNAPSHOT=${TESTDIR}/snapshot
+BUILD=${TESTDIR}/template/build
+
+
+# copy all the build output to a snapshot directory
+function take_build_snapshot()
+{
+	rm -rf ${SNAPSHOT}
+	cp -ap ${TESTDIR}/template/build ${SNAPSHOT}
+}
+
+# verify that all the arguments are present in the build output directory
+function assert_built()
+{
+	until [ -z "$1" ]; do
+		if [ ! -f "${BUILD}/$1" ]; then
+			failure "File $1 should be in the build output directory"
+		fi
+		shift
+	done
+}
+
+# verify all the arguments passed in were rebuilt relative to the snapshot
+function assert_rebuilt()
+{
+	until [ -z "$1" ]; do
+		assert_built "$1"
+		if [ ! -f "${SNAPSHOT}/$1" ]; then
+			failure "File $1 should have been original build snapshot"
+		fi
+		if [ ! "${SNAPSHOT}/$1" -ot "${BUILD}/$1" ]; then
+			failure "File $1 should have been rebuilt"
+		fi
+		shift
+	done
+}
+
+# verify all the arguments are in the build directory & snapshot,
+# but were not rebuilt
+function assert_not_rebuilt()
+{
+	until [ -z "$1" ]; do
+		assert_built "$1"
+		if [ ! -f "${SNAPSHOT}/$1" ]; then
+			failure "File $1 should be in snapshot build directory"
+		fi
+		if [ "${SNAPSHOT}/$1" -ot "${BUILD}/$1" ]; then
+			failure "File $1 should not have been rebuilt"
+		fi
+		shift
+	done
+}
+
+cd ${TESTDIR}
+run_tests

make/common.mk

@@ -35,13 +35,18 @@ CXXFLAGS = -DESP_PLATFORM -Og -std=gnu++11 -g3 \
 endif
 
 #Handling of V=1/VERBOSE=1 flag
+#
+# if V=1, $(summary) does nothing and $(details) will echo extra details
+# if V is unset or not 1, $(summary) echoes a summary and $(details) does nothing
 V ?= $(VERBOSE)
 ifeq ("$(V)","1")
 Q :=
-vecho := @true
+summary := @true
+details := @echo
 else
 Q := @
-vecho := @echo
+summary := @echo
+details := @true
 endif
 
 # General make utilities

make/component.mk

@@ -71,14 +71,14 @@ build: $(COMPONENT_LIBRARY)
 #Build the archive. We remove the archive first, otherwise ar will get confused if we update
 #an archive when multiple filenames have the same name (src1/test.o and src2/test.o)
 $(COMPONENT_LIBRARY): $(COMPONENT_OBJS)
-	$(vecho) AR $@
+	$(summary) AR $@
 	$(Q) rm -f $@
 	$(Q) $(AR) cru $@ $(COMPONENT_OBJS)
 endif
 
 ifeq ("$(COMPONENT_OWNCLEANTARGET)", "")
 clean:
-	$(vecho) RM $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EXTRA_CLEAN)
+	$(summary) RM $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EXTRA_CLEAN)
 	$(Q) rm -f $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EXTRA_CLEAN)
 endif
 
@@ -92,15 +92,15 @@ CXXFLAGS+=-MMD
 define GenerateCompileTargets
 # $(1) - directory containing source files, relative to $(COMPONENT_PATH)
 $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.c | $(1)
-	$$(vecho) CC $$@
+	$$(summary) CC $$@
 	$$(Q) $$(CC) $$(CFLAGS) $$(addprefix -I ,$$(COMPONENT_INCLUDES)) $$(addprefix -I ,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@
 
 $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.cpp | $(1)
-	$$(vecho) CC $$@
+	$$(summary) CC $$@
 	$$(Q) $$(CXX) $$(CXXFLAGS) $$(addprefix -I,$$(COMPONENT_INCLUDES)) $$(addprefix -I,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@
 
 $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.S | $(1)
-	$$(vecho) CC $$@
+	$$(summary) CC $$@
 	$$(Q) $$(CC) $$(CFLAGS) $$(addprefix -I ,$$(COMPONENT_INCLUDES)) $$(addprefix -I ,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@
 
 # CWD is build dir, create the build subdirectory if it doesn't exist

make/project.mk

@@ -166,9 +166,15 @@ $(foreach componentpath,$(COMPONENT_PATHS),$(eval $(call includeProjBuildMakefil
 # once we know component paths, we can include the config
 include $(IDF_PATH)/make/project_config.mk
 
-# ELF depends on the -build target of every component
+# A "component" library is any library in the LDFLAGS where
-$(APP_ELF): $(addsuffix -build,$(notdir $(COMPONENT_PATHS_BUILDABLE)))
+# the name of the library is also a name of the component
-	$(vecho) LD $(notdir $@)
+APP_LIBRARIES = $(patsubst -l%,%,$(filter -l%,$(LDFLAGS)))
+COMPONENT_LIBRARIES = $(filter $(notdir $(COMPONENT_PATHS_BUILDABLE)),$(APP_LIBRARIES))
+
+# ELF depends on the library archive files for COMPONENT_LIBRARIES
+# the rules to build these are emitted as part of GenerateComponentTarget below
+$(APP_ELF): $(foreach libcomp,$(COMPONENT_LIBRARIES),$(BUILD_DIR_BASE)/$(libcomp)/lib$(libcomp).a)
+	$(summary) LD $(notdir $@)
 	$(Q) $(CC) $(LDFLAGS) -o $@ -Wl,-Map=$(APP_MAP)
 
 # Generation of $(APP_BIN) from $(APP_ELF) is added by the esptool
@@ -182,28 +188,34 @@ all_binaries: $(APP_BIN)
 $(BUILD_DIR_BASE):
 	mkdir -p $(BUILD_DIR_BASE)
 
-define GenerateComponentTarget
+define GenerateComponentPhonyTarget
 # $(1) - path to component dir
 # $(2) - target to generate (build, clean)
-# $(3) - optional dependencies to add
 .PHONY: $(notdir $(1))-$(2)
-$(notdir $(1))-$(2): $(3) | $(BUILD_DIR_BASE)/$(notdir $(1))
+$(notdir $(1))-$(2): | $(BUILD_DIR_BASE)/$(notdir $(1))
 	@+$(MAKE) -C $(BUILD_DIR_BASE)/$(notdir $(1)) -f $(1)/Makefile COMPONENT_BUILD_DIR=$(BUILD_DIR_BASE)/$(notdir $(1)) $(2)
 endef
 
-define GenerateComponentBuildDirTarget
+define GenerateComponentTargets
 # $(1) - path to component dir
 $(BUILD_DIR_BASE)/$(notdir $(1)):
 	@mkdir -p $(BUILD_DIR_BASE)/$(notdir $(1))
+
+# tell make it can build any component's library by invoking the recursive -build target
+# (this target exists for all components even ones which don't build libraries, but it's
+# only invoked for the targets whose libraries appear in COMPONENT_LIBRARIES and hence the
+# APP_ELF dependencies.)
+$(BUILD_DIR_BASE)/$(notdir $(1))/lib$(notdir $(1)).a: $(notdir $(1))-build
+	$(details) echo "$$^ responsible for $$@" # echo which build target built this file
 endef
 
-$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentBuildDirTarget,$(component))))
+$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentTargets,$(component))))
 
-$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentTarget,$(component),build,$(PROJECT_PATH)/build/include/sdkconfig.h)))
+$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentPhonyTarget,$(component),build)))
-$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentTarget,$(component),clean)))
+$(foreach component,$(COMPONENT_PATHS_BUILDABLE),$(eval $(call GenerateComponentPhonyTarget,$(component),clean)))
 
 app-clean: $(addsuffix -clean,$(notdir $(COMPONENT_PATHS_BUILDABLE)))
-	$(vecho) RM $(APP_ELF)
+	$(summary) RM $(APP_ELF)
 	$(Q) rm -f $(APP_ELF) $(APP_BIN) $(APP_MAP)
 
 clean: app-clean

make/project_config.mk

@@ -16,7 +16,7 @@ $(KCONFIG_TOOL_DIR)/mconf $(KCONFIG_TOOL_DIR)/conf:
 	$(MAKE) -C $(KCONFIG_TOOL_DIR)
 
 menuconfig: $(KCONFIG_TOOL_DIR)/mconf $(IDF_PATH)/Kconfig $(BUILD_DIR_BASE)
-	$(vecho) MENUCONFIG
+	$(summary) MENUCONFIG
 	$(Q) KCONFIG_AUTOHEADER=$(PROJECT_PATH)/build/include/sdkconfig.h \
 	KCONFIG_CONFIG=$(PROJECT_PATH)/sdkconfig \
 	COMPONENT_KCONFIGS="$(COMPONENT_KCONFIGS)" \
@@ -29,7 +29,7 @@ $(PROJECT_PATH)/sdkconfig: menuconfig
 endif
 
 defconfig: $(KCONFIG_TOOL_DIR)/mconf $(IDF_PATH)/Kconfig $(BUILD_DIR_BASE)
-	$(vecho) DEFCONFIG
+	$(summary) DEFCONFIG
 	$(Q) mkdir -p $(PROJECT_PATH)/build/include/config
 	$(Q) KCONFIG_AUTOHEADER=$(PROJECT_PATH)/build/include/sdkconfig.h \
 	KCONFIG_CONFIG=$(PROJECT_PATH)/sdkconfig \
@@ -53,7 +53,7 @@ endif
 endif
 
 $(AUTO_CONF_REGEN_TARGET) $(PROJECT_PATH)/build/include/sdkconfig.h: $(PROJECT_PATH)/sdkconfig $(KCONFIG_TOOL_DIR)/conf $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD)
-	$(vecho) GENCONFIG
+	$(summary) GENCONFIG
 	$(Q) mkdir -p $(PROJECT_PATH)/build/include/config
 	$(Q) cd build; KCONFIG_AUTOHEADER="$(PROJECT_PATH)/build/include/sdkconfig.h" \
 	KCONFIG_CONFIG=$(PROJECT_PATH)/sdkconfig \
@@ -68,6 +68,6 @@ $(AUTO_CONF_REGEN_TARGET) $(PROJECT_PATH)/build/include/sdkconfig.h: $(PROJECT_P
 clean: config-clean
 .PHONY: config-clean
 config-clean:
-	$(vecho RM CONFIG)
+	$(summary RM CONFIG)
 	$(MAKE) -C $(KCONFIG_TOOL_DIR) clean
 	$(Q) rm -rf $(PROJECT_PATH)/build/include/config $(PROJECT_PATH)/build/include/sdkconfig.h