Merge branch 'feature/ci_fuzzer_tests_lwip' into 'master'

ci jobs running fuzz tests on dns, dhcp, dhcps + removed unnecessary references (libexpat)

See merge request idf/esp-idf!2866

.gitlab-ci.yml

@@ -303,7 +303,7 @@ test_fatfs_on_host:
     - cd components/fatfs/test_fatfs_host/
     - make test
 
-test_mdns_fuzzer_on_host:
+.host_fuzzer_test_template: &host_fuzzer_test_template
   stage: host_test
   image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
   tags:
@@ -312,7 +312,8 @@ test_mdns_fuzzer_on_host:
   artifacts:
     when: always
     paths:
-      - components/mdns/test_afl_fuzz_host/out/crashes
+      - ${FUZZER_TEST_DIR}/out/crashes
+      - ${FUZZER_TEST_DIR}/fuzz_output.log
     expire_in: 1 mos
   only:
     # can only be triggered
@@ -321,12 +322,35 @@ test_mdns_fuzzer_on_host:
     BOT_NEEDS_TRIGGER_BY_NAME: 1
   script:
     - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
-    - cd components/mdns/test_afl_fuzz_host/
+    - cd ${FUZZER_TEST_DIR}
     # run AFL fuzzer for one hour
-    - ( make fuzz || pkill sleep ) &
+    - ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
     - ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
     # check no crashes found
-    - "[ -z `ls out/crashes/` ] || exit 1"
+    - test -z "$(ls out/crashes/)" || exit 1
+
+test_mdns_fuzzer_on_host:
+  <<: *host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/mdns/test_afl_fuzz_host
+
+test_lwip_dns_fuzzer_on_host:
+  <<: *host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dns
+
+test_lwip_dhcp_fuzzer_on_host:
+  <<: *host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dhcp_client
+
+test_lwip_dhcps_fuzzer_on_host:
+  <<: *host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dhcp_server
 
 test_spiffs_on_host:
   <<: *host_test_template

components/lwip/core/ipv4/dhcp.c

@@ -1712,7 +1712,7 @@ decode_next:
       offset_max -= q->len;
       if ((offset < offset_max) && offset_max) {
         q = q->next;
-        LWIP_ASSERT("next pbuf was null", q);
+        LWIP_ERROR("offset pointed to next pbuf which is null", q , return ERR_VAL;);
         options = (u8_t*)q->payload;
       } else {
         /* We've run out of bytes, probably no end marker. Don't proceed. */

components/lwip/test_afl_host/Makefile

@@ -1,7 +1,7 @@
 COMPONENTS_DIR=../..
 CFLAGS=-std=gnu99 -Og -ggdb -ffunction-sections -fdata-sections -nostdlib -Wall  -Werror=all -Wno-int-to-pointer-cast -Wno-error=unused-function -Wno-error=unused-variable -Wno-error=deprecated-declarations -Wextra \
 -Wno-unused-parameter -Wno-sign-compare -Wno-address   -Wno-unused-variable -DESP_PLATFORM -D IDF_VER=\"v3.1\" -MMD -MP -DWITH_POSIX
-INC_DIRS=-I . -I $(COMPONENTS_DIR)/lwip/include/lwip -I $(COMPONENTS_DIR)/lwip/include/lwip/port -I $(COMPONENTS_DIR)/lwip/include/lwip/posix -I $(COMPONENTS_DIR)/lwip/apps/ping -I $(COMPONENTS_DIR)/app_trace/include -I $(COMPONENTS_DIR)/app_update/include -I $(COMPONENTS_DIR)/bootloader_support/include -I $(COMPONENTS_DIR)/bt/include -I $(COMPONENTS_DIR)/coap/port/include -I $(COMPONENTS_DIR)/coap/port/include/coap -I $(COMPONENTS_DIR)/coap/libcoap/include -I \ $(COMPONENTS_DIR)/coap/libcoap/include/coap -I $(COMPONENTS_DIR)/console -I $(COMPONENTS_DIR)/cxx/include -I $(COMPONENTS_DIR)/driver/include -I $(COMPONENTS_DIR)/esp-tls -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/esp_adc_cal/include -I $(COMPONENTS_DIR)/ethernet/include -I $(COMPONENTS_DIR)/expat/port/include -I $(COMPONENTS_DIR)/expat/include/expat -I $(COMPONENTS_DIR)/fatfs/src -I $(COMPONENTS_DIR)/freertos/include -I $(COMPONENTS_DIR)/heap/include -I \ $(COMPONENTS_DIR)/idf_test/include -I $(COMPONENTS_DIR)/jsmn/include -I $(COMPONENTS_DIR)/json/cJSON -I $(COMPONENTS_DIR)/libsodium/libsodium/src/libsodium/include -I $(COMPONENTS_DIR)/libsodium/port_include -I $(COMPONENTS_DIR)/log/include -I /home/david/esp/esp-idf/examples/wifi/simple_wifi/main/include -I $(COMPONENTS_DIR)/mbedtls/port/include -I $(COMPONENTS_DIR)/mbedtls/include -I $(COMPONENTS_DIR)/mdns/include -I $(COMPONENTS_DIR)/micro-ecc/micro-ecc -I \ $(COMPONENTS_DIR)/newlib/platform_include -I $(COMPONENTS_DIR)/newlib/include -I $(COMPONENTS_DIR)/nghttp/port/include -I $(COMPONENTS_DIR)/nghttp/nghttp2/lib/includes -I $(COMPONENTS_DIR)/nvs_flash/include -I $(COMPONENTS_DIR)/openssl/include -I $(COMPONENTS_DIR)/pthread/include -I $(COMPONENTS_DIR)/sdmmc/include -I $(COMPONENTS_DIR)/smartconfig/include -I $(COMPONENTS_DIR)/soc/esp32/include -I $(COMPONENTS_DIR)/soc/include -I $(COMPONENTS_DIR)/spi_flash/include -I \ $(COMPONENTS_DIR)/spiffs/include -I $(COMPONENTS_DIR)/tcpip_adapter/include -I $(COMPONENTS_DIR)/ulp/include -I $(COMPONENTS_DIR)/vfs/include -I $(COMPONENTS_DIR)/wear_levelling/include -I $(COMPONENTS_DIR)/wpa_supplicant/include -I $(COMPONENTS_DIR)/wpa_supplicant/port/include -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/xtensa-debug-module/include
+INC_DIRS=-I . -I $(COMPONENTS_DIR)/newlib/platform_include -I $(COMPONENTS_DIR)/newlib/include -I $(COMPONENTS_DIR)/driver/include -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/ethernet/include -I $(COMPONENTS_DIR)/freertos/include -I $(COMPONENTS_DIR)/heap/include -I $(COMPONENTS_DIR)/lwip/include/lwip -I $(COMPONENTS_DIR)/lwip/include/lwip/port -I $(COMPONENTS_DIR)/lwip/include/lwip/posix -I $(COMPONENTS_DIR)/lwip/apps/ping -I $(COMPONENTS_DIR)/soc/esp32/include -I $(COMPONENTS_DIR)/soc/include -I $(COMPONENTS_DIR)/tcpip_adapter/include  -I $(COMPONENTS_DIR)/xtensa-debug-module/include
 TEST_NAME=test
 FUZZ=afl-fuzz
 LD=$(CC)
@@ -55,4 +55,4 @@ $(TEST_NAME): $(OBJECTS)
 	@$(LD)  $(OBJECTS) -o $@ $(LDLIBS)
 
 fuzz: $(TEST_NAME)
-	@$(FUZZ) -t 500 -i "$(SAMPLE_PACKETS)" -o "out" -- ./$(TEST_NAME)
+	@$(FUZZ) -t 5000+ -i "$(SAMPLE_PACKETS)" -o "out" -- ./$(TEST_NAME)

components/lwip/test_afl_host/test_dhcp_server.c

@@ -9,6 +9,9 @@ const ip_addr_t ip_addr_any;
 ip4_addr_t server_ip;
 struct netif mynetif;
 
+// dhcps callback
+void dhcp_test_dhcps_cb (u8_t client_ip[4]) {}
+
 // Dependency injected static function to pass the packet into parser
 void dhcp_test_handle_dhcp(void *arg, struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *addr, u16_t port);
 void dhcp_test_init_di();
@@ -26,6 +29,7 @@ int main(int argc, char** argv)
     dhcp_test_init_di();
 
     IP4_ADDR(&server_ip, 192,168,4,1);
+    dhcps_set_new_lease_cb(dhcp_test_dhcps_cb);
     dhcps_start(&mynetif, server_ip);
 
 #ifdef INSTR_IS_OFF