CI: public header check script in python
Additional test compiling the header to produce assembly: - checks if header really compiles (previously validated only by preprocessor) - checks that the header produced empty object, no inline code to be present in public headers Closes IDF-1324
This commit is contained in:
David Cermak
parent
0ae960f2fe
commit
8f6645667e
328
tools/ci/check_public_headers.py
Normal file
328
tools/ci/check_public_headers.py
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Checks all public headers in IDF in the ci
|
||||
#
|
||||
# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
import re
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
import fnmatch
|
||||
import argparse
|
||||
import queue
|
||||
from threading import Thread, Event
|
||||
import tempfile
|
||||
|
||||
|
||||
class HeaderFailed(Exception):
|
||||
"""Base header failure exeption"""
|
||||
pass
|
||||
|
||||
|
||||
class HeaderFailedSdkconfig(HeaderFailed):
|
||||
def __str__(self):
|
||||
return "Sdkconfig Error"
|
||||
|
||||
|
||||
class HeaderFailedBuildError(HeaderFailed):
|
||||
def __str__(self):
|
||||
return "Header Build Error"
|
||||
|
||||
|
||||
class HeaderFailedCppGuardMissing(HeaderFailed):
|
||||
def __str__(self):
|
||||
return "Header Missing C++ Guard"
|
||||
|
||||
|
||||
class HeaderFailedContainsCode(HeaderFailed):
|
||||
def __str__(self):
|
||||
return "Header Produced non-zero object"
|
||||
|
||||
|
||||
# Creates a temp file and returns both output as a string and a file name
|
||||
#
|
||||
def exec_cmd_to_temp_file(what, suffix=""):
|
||||
out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
|
||||
rc, out, err = exec_cmd(what, out_file)
|
||||
with open(out_file.name, "r") as f:
|
||||
out = f.read()
|
||||
return rc, out, err, out_file.name
|
||||
|
||||
|
||||
def exec_cmd(what, out_file=subprocess.PIPE):
|
||||
p = subprocess.Popen(what, stdin=subprocess.PIPE, stdout=out_file, stderr=subprocess.PIPE)
|
||||
output, err = p.communicate()
|
||||
rc = p.returncode
|
||||
output = output.decode('utf-8') if output is not None else None
|
||||
err = err.decode('utf-8') if err is not None else None
|
||||
return rc, output, err
|
||||
|
||||
|
||||
class PublicHeaderChecker:
|
||||
# Intermediate results
|
||||
COMPILE_ERR_REF_CONFIG_HDR_FAILED = 1 # -> Cannot compile and failed with injected SDKCONFIG #error (header FAILs)
|
||||
COMPILE_ERR_ERROR_MACRO_HDR_OK = 2 # -> Cannot compile, but failed with "#error" directive (header seems OK)
|
||||
COMPILE_ERR_HDR_FAILED = 3 # -> Cannot compile with another issue, logged if verbose (header FAILs)
|
||||
PREPROC_OUT_ZERO_HDR_OK = 4 # -> Both preprocessors produce zero out (header file is OK)
|
||||
PREPROC_OUT_SAME_HRD_FAILED = 5 # -> Both preprocessors produce the same, non-zero output (header file FAILs)
|
||||
PREPROC_OUT_DIFFERENT_WITH_EXT_C_HDR_OK = 6 # -> Both preprocessors produce different, non-zero output with extern "C" (header seems OK)
|
||||
PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED = 7 # -> Both preprocessors produce different, non-zero output without extern "C" (header fails)
|
||||
|
||||
def log(self, message):
|
||||
if self.verbose:
|
||||
print(message)
|
||||
|
||||
def __init__(self, verbose=False, jobs=1, prefix=None):
|
||||
self.gcc = "{}gcc".format(prefix)
|
||||
self.gpp = "{}g++".format(prefix)
|
||||
self.verbose = verbose
|
||||
self.jobs = jobs
|
||||
self.prefix = prefix
|
||||
self.extern_c = re.compile(r'extern "C"')
|
||||
self.error_macro = re.compile(r'#error')
|
||||
self.error_orphan_kconfig = re.compile(r"#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED")
|
||||
self.kconfig_macro = re.compile(r'\bCONFIG_[A-Z0-9_]+')
|
||||
self.assembly_nocode = r'^\s*(\.file|\.text|\.ident).*$'
|
||||
self.check_threads = []
|
||||
|
||||
self.job_queue = queue.Queue()
|
||||
self.failed_queue = queue.Queue()
|
||||
self.terminate = Event()
|
||||
|
||||
def __enter__(self):
|
||||
for i in range(self.jobs):
|
||||
t = Thread(target=self.check_headers, args=(i, ))
|
||||
self.check_threads.append(t)
|
||||
t.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.terminate.set()
|
||||
for t in self.check_threads:
|
||||
t.join()
|
||||
|
||||
# thread function process incoming header file from a queue
|
||||
def check_headers(self, num):
|
||||
while not self.terminate.is_set():
|
||||
if not self.job_queue.empty():
|
||||
task = self.job_queue.get()
|
||||
if task is None:
|
||||
self.terminate.set()
|
||||
else:
|
||||
try:
|
||||
self.check_one_header(task, num)
|
||||
except HeaderFailed as e:
|
||||
self.failed_queue.put("{}: Failed! {}".format(task, e))
|
||||
|
||||
def get_failed(self):
|
||||
return list(self.failed_queue.queue)
|
||||
|
||||
def join(self):
|
||||
for t in self.check_threads:
|
||||
while t.isAlive and not self.terminate.is_set():
|
||||
t.join(1) # joins with timeout to respond to keyboard interrupt
|
||||
|
||||
# Checks one header calling:
|
||||
# - preprocess_one_header() to test and compare preprocessor outputs
|
||||
# - check_no_code() to test if header contains some executable code
|
||||
# Procedure
|
||||
# 1) Preprocess the include file with C preprocessor and with CPP preprocessor
|
||||
# - Pass the test if the preprocessor outputs are the same and whitespaces only (#define only header)
|
||||
# - Fail the test if the preprocessor outputs are the same (but with some code)
|
||||
# - If outputs different, continue with 2)
|
||||
# 2) Strip out all include directives to generate "temp.h"
|
||||
# 3) Preprocess the temp.h the same way in (1)
|
||||
# - Pass the test if the preprocessor outputs are the same and whitespaces only (#include only header)
|
||||
# - Fail the test if the preprocessor outputs are the same (but with some code)
|
||||
# - If outputs different, pass the test
|
||||
# 4) If header passed the steps 1) and 3) test that it produced zero assembly code
|
||||
def check_one_header(self, header, num):
|
||||
res = self.preprocess_one_header(header, num)
|
||||
if res == self.COMPILE_ERR_REF_CONFIG_HDR_FAILED:
|
||||
raise HeaderFailedSdkconfig()
|
||||
elif res == self.COMPILE_ERR_ERROR_MACRO_HDR_OK:
|
||||
return self.compile_one_header(header)
|
||||
elif res == self.COMPILE_ERR_HDR_FAILED:
|
||||
raise HeaderFailedBuildError()
|
||||
elif res == self.PREPROC_OUT_ZERO_HDR_OK:
|
||||
return self.compile_one_header(header)
|
||||
elif res == self.PREPROC_OUT_SAME_HRD_FAILED:
|
||||
raise HeaderFailedCppGuardMissing()
|
||||
else:
|
||||
self.compile_one_header(header)
|
||||
try:
|
||||
_, _, _, temp_header = exec_cmd_to_temp_file(["sed", "/#include/d; /#error/d", header], suffix=".h")
|
||||
res = self.preprocess_one_header(temp_header, num, ignore_sdkconfig_issue=True)
|
||||
if res == self.PREPROC_OUT_SAME_HRD_FAILED:
|
||||
raise HeaderFailedCppGuardMissing()
|
||||
elif res == self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED:
|
||||
raise HeaderFailedCppGuardMissing()
|
||||
finally:
|
||||
os.unlink(temp_header)
|
||||
|
||||
def compile_one_header(self, header):
|
||||
rc, out, err = exec_cmd([self.gcc, "-S", "-o-", "-include", header, self.main_c] + self.include_dir_flags)
|
||||
if rc == 0:
|
||||
if not re.sub(self.assembly_nocode, '', out, flags=re.M).isspace():
|
||||
raise HeaderFailedContainsCode()
|
||||
return # Header OK: produced zero code
|
||||
raise HeaderFailedBuildError()
|
||||
|
||||
def preprocess_one_header(self, header, num, ignore_sdkconfig_issue=False):
|
||||
all_compilation_flags = ["-w", "-P", "-E", "-include", header, self.main_c] + self.include_dir_flags
|
||||
if not ignore_sdkconfig_issue:
|
||||
# just strip commnets to check for CONFIG_... macros
|
||||
rc, out, err = exec_cmd([self.gcc, "-fpreprocessed", "-dD", "-P", "-E", header] + self.include_dir_flags)
|
||||
if re.search(self.kconfig_macro, out):
|
||||
# enable defined #error if sdkconfig.h not included
|
||||
all_compilation_flags.append("-DIDF_CHECK_SDKCONFIG_INCLUDED")
|
||||
try:
|
||||
# compile with C++, check for errors, outputs for a temp file
|
||||
rc, cpp_out, err, cpp_out_file = exec_cmd_to_temp_file([self.gpp, "--std=c++17"] + all_compilation_flags)
|
||||
if rc != 0:
|
||||
if re.search(self.error_macro, err):
|
||||
if re.search(self.error_orphan_kconfig, err):
|
||||
self.log("{}: CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED".format(header))
|
||||
return self.COMPILE_ERR_REF_CONFIG_HDR_FAILED
|
||||
self.log("{}: Error directive failure: OK".format(header))
|
||||
return self.COMPILE_ERR_ERROR_MACRO_HDR_OK
|
||||
self.log("{}: FAILED: compilation issue".format(header))
|
||||
self.log(err)
|
||||
return self.COMPILE_ERR_HDR_FAILED
|
||||
# compile with C compiler, outputs to another temp file
|
||||
rc, c99_out, err, c99_out_file = exec_cmd_to_temp_file([self.gcc, "--std=c99"] + all_compilation_flags)
|
||||
if rc != 0:
|
||||
self.log("{} FAILED should never happen".format(header))
|
||||
return self.COMPILE_ERR_HDR_FAILED
|
||||
# diff the two outputs
|
||||
rc, diff, err = exec_cmd(["diff", c99_out_file, cpp_out_file])
|
||||
if not diff or diff.isspace():
|
||||
if not cpp_out or cpp_out.isspace():
|
||||
self.log("{} The same, but empty out - OK".format(header))
|
||||
return self.PREPROC_OUT_ZERO_HDR_OK
|
||||
self.log("{} FAILED It is the same!".format(header))
|
||||
return self.PREPROC_OUT_SAME_HRD_FAILED
|
||||
if re.search(self.extern_c, diff):
|
||||
self.log("{} extern C present - OK".format(header))
|
||||
return self.PREPROC_OUT_DIFFERENT_WITH_EXT_C_HDR_OK
|
||||
self.log("{} Different but no extern C - FAILED".format(header))
|
||||
return self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED
|
||||
finally:
|
||||
os.unlink(cpp_out_file)
|
||||
try:
|
||||
os.unlink(c99_out_file)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Get compilation data from an example to list all public header files
|
||||
def list_public_headers(self, ignore_dirs, ignore_files):
|
||||
idf_path = os.getenv('IDF_PATH')
|
||||
project_dir = os.path.join(idf_path, "examples", "get-started", "blink")
|
||||
subprocess.check_call(["idf.py", "reconfigure"], cwd=project_dir)
|
||||
build_commands_json = os.path.join(project_dir, "build", "compile_commands.json")
|
||||
with open(build_commands_json, "r") as f:
|
||||
build_command = json.load(f)[0]["command"].split()
|
||||
include_dir_flags = []
|
||||
include_dirs = []
|
||||
# process compilation flags (includes and defines)
|
||||
for item in build_command:
|
||||
if item.startswith("-I"):
|
||||
include_dir_flags.append(item)
|
||||
if "components" in item:
|
||||
include_dirs.append(item[2:]) # Removing the leading "-I"
|
||||
if item.startswith("-D"):
|
||||
include_dir_flags.append(item.replace('\\','')) # removes escaped quotes, eg: -DMBEDTLS_CONFIG_FILE=\\\"mbedtls/esp_config.h\\\"
|
||||
include_dir_flags.append("-I" + os.path.join(project_dir, "build", "config"))
|
||||
sdkconfig_h = os.path.join(project_dir, "build", "config", "sdkconfig.h")
|
||||
# prepares a main_c file for easier sdkconfig checks and avoid compilers warning when compiling headers directly
|
||||
with open(sdkconfig_h, "a") as f:
|
||||
f.write("#define IDF_SDKCONFIG_INCLUDED")
|
||||
main_c = os.path.join(project_dir, "build", "compile.c")
|
||||
with open(main_c, "w") as f:
|
||||
f.write("#if defined(IDF_CHECK_SDKCONFIG_INCLUDED) && ! defined(IDF_SDKCONFIG_INCLUDED)\n"
|
||||
"#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED\n"
|
||||
"#endif")
|
||||
# processes public include dirs, removing ignored files
|
||||
all_include_files = []
|
||||
files_to_check = []
|
||||
for d in include_dirs:
|
||||
if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)):
|
||||
self.log("{} - directory ignored".format(d))
|
||||
continue
|
||||
for root, dirnames, filenames in os.walk(d):
|
||||
for filename in fnmatch.filter(filenames, '*.h'):
|
||||
all_include_files.append(os.path.join(root, filename))
|
||||
self.main_c = main_c
|
||||
self.include_dir_flags = include_dir_flags
|
||||
ignore_files = set(ignore_files)
|
||||
# processes public include files, removing ignored files
|
||||
for f in all_include_files:
|
||||
rel_path_file = os.path.relpath(f, idf_path)
|
||||
if any([os.path.commonprefix([d, rel_path_file]) == d for d in ignore_dirs]):
|
||||
self.log("{} - file ignored (inside ignore dir)".format(f))
|
||||
continue
|
||||
if rel_path_file in ignore_files:
|
||||
self.log("{} - file ignored".format(f))
|
||||
continue
|
||||
files_to_check.append(f)
|
||||
# removes duplicates and places headers to a work queue
|
||||
for f in set(files_to_check):
|
||||
self.job_queue.put(f)
|
||||
self.job_queue.put(None) # to indicate the last job
|
||||
|
||||
|
||||
def check_all_headers():
|
||||
parser = argparse.ArgumentParser("Public header checker file")
|
||||
parser.add_argument("--verbose", "-v", help="enables verbose mode", action="store_true")
|
||||
parser.add_argument("--jobs", "-j", help="number of jobs to run checker", default=1, type=int)
|
||||
parser.add_argument("--prefix", "-p", help="compiler prefix", default="xtensa-esp32-elf-", type=str)
|
||||
parser.add_argument("--exclude-file", "-e", help="exception file", default="check_public_headers_exceptions.txt", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
# process excluded files and dirs
|
||||
exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file)
|
||||
with open(exclude_file, "r") as f:
|
||||
lines = [line.rstrip() for line in f]
|
||||
ignore_files = []
|
||||
ignore_dirs = []
|
||||
for line in lines:
|
||||
if not line or line.isspace() or line.startswith("#"):
|
||||
continue
|
||||
if os.path.isdir(line):
|
||||
ignore_dirs.append(line)
|
||||
else:
|
||||
ignore_files.append(line)
|
||||
|
||||
# start header check
|
||||
with PublicHeaderChecker(args.verbose, args.jobs, args.prefix) as header_check:
|
||||
header_check.list_public_headers(ignore_dirs, ignore_files)
|
||||
try:
|
||||
header_check.join()
|
||||
failures = header_check.get_failed()
|
||||
if len(failures) > 0:
|
||||
for failed in failures:
|
||||
print(failed)
|
||||
exit(1)
|
||||
print("No errors found")
|
||||
except KeyboardInterrupt:
|
||||
print("Keyboard interrupt")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
check_all_headers()
|
||||
|
|
@ -1,391 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# This script finds all *.h files in public include dirs and checks for
|
||||
# * compilation issues
|
||||
# * c++ header guards
|
||||
# * need for sdkconfig.h inclusion (if CONFIG_ macros referenced)
|
||||
#
|
||||
# Script arguments:
|
||||
# --verbose: enables printing details if compilation fails (verbose is off by default)
|
||||
# --process-all-files: all files are checked and then excluded, by default the excluded files are skipped and not checked
|
||||
# --include-maxdepth=X: maximum directory depth for finding include files in include dirs (by default not specified=infinity)
|
||||
# --jobs=X: run the checker in X separate processes
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ -z "${IDF_PATH:-}" ]; then
|
||||
echo "Please run export.sh before invoking this script"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
##################### helper functions #####################
|
||||
|
||||
# log_error <filename>
|
||||
#
|
||||
# Logs details on filename processing when the include fails
|
||||
function log_error()
|
||||
{
|
||||
if [ ! -z "${VERBOSE:-}" ]; then
|
||||
file=$1
|
||||
echo " verbose..."
|
||||
echo "$LINE"
|
||||
grep -A5 "error:"
|
||||
echo "$LINE"
|
||||
echo -n "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
# log_details <log_data>
|
||||
#
|
||||
# Logs details as function argument to stdout
|
||||
function log_details()
|
||||
{
|
||||
if [ ! -z "${VERBOSE:-}" ]; then
|
||||
printf "... $1 "
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
# preproc_one_header <filename>
|
||||
#
|
||||
# Function to preprocess a header with CC and CXX and compares result
|
||||
# Returns
|
||||
# 0 -> Never returns zero (for easier return value expansion)
|
||||
# 1 -> Cannot compile and failed with injected SDKCONFIG #error (header FAILs)
|
||||
# 2 -> Cannot compile, but failed with "#error" directive (header seems OK)
|
||||
# 3 -> Cannot compile with another issue, logged if verbose (header FAILs)
|
||||
# 4 -> Both preprocessors produce zero out (header file is OK)
|
||||
# 5 -> Both preprocessors produce the same, non-zero output (header file FAILs)
|
||||
# 6 -> Both preprocessors produce different, non-zero output with extern "C" (header seems OK)
|
||||
# 7 -> Both preprocessors produce different, non-zero output without extern "C" (header fails)
|
||||
function preproc_one_header()
|
||||
{
|
||||
local extra_flags=""
|
||||
local file=$1
|
||||
# check if the header (after removing comments) references CONFIG_XXX macros
|
||||
if ${CC} -fpreprocessed -dD -P -E ${file} ${FLAGS_INCLUDE_DIRS} 2>/dev/null | grep -q -wo -E 'CONFIG_[A-Z0-9_]+'; then
|
||||
# enable sdkconfig.h inclusion checker
|
||||
extra_flags="-DIDF_CHECK_SDKCONFIG_INCLUDED"
|
||||
fi
|
||||
if ! cpp_out=`${CXX} -w -P -E --std=c++17 -include ${file} ${extra_flags} ${TEST_MAIN_C} ${FLAGS_INCLUDE_DIRS} 2>&1`; then
|
||||
if [[ "$cpp_out" =~ "#error" ]]; then
|
||||
if [[ "$cpp_out" =~ "#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED" ]]; then
|
||||
# failed for sdkconfig.h issue
|
||||
return 1
|
||||
fi;
|
||||
# header passed in general (error directive)
|
||||
return 2
|
||||
fi;
|
||||
# Logging the compilation issue
|
||||
echo "$cpp_out" | log_error $file
|
||||
return 3
|
||||
fi;
|
||||
if ! c99_out=`${CC} -w -P -E --std=c99 -include ${file} ${extra_flags} ${TEST_MAIN_C} ${FLAGS_INCLUDE_DIRS} 2>/dev/null`; then
|
||||
# Should not happen, fail with compilation issue
|
||||
return 3
|
||||
fi;
|
||||
if diff_out=`diff <(echo "$cpp_out" ) <(echo "$c99_out")`; then
|
||||
# checks zero output: contains spaces only if lines < 5 (for performace)
|
||||
lines=`echo "$cpp_out" | wc -l`
|
||||
if [ $lines -lt 5 ] && [ -z "${cpp_out// }" ]; then
|
||||
return 4
|
||||
fi
|
||||
return 5
|
||||
elif echo $diff_out | grep -q 'extern "C"'; then
|
||||
return 6
|
||||
fi
|
||||
return 7
|
||||
}
|
||||
|
||||
# check_one_header <filename>
|
||||
#
|
||||
# Function used to implement the logic of checking a single header.
|
||||
# Returns 0 on success (as a normal bash function would).
|
||||
# - returns 1 if suspects C++ guard missing
|
||||
# - returns 2 if compile issue (other then error directive)
|
||||
# - returns 3 if CONFIG_XXX variable used but sdkconfig.h not included
|
||||
# Ideally shouldn't print anything (if verbose, preproc_one_header outputs compilation errors)
|
||||
function check_one_header
|
||||
{
|
||||
include_file=$1
|
||||
# Procedure:
|
||||
# 1) Preprocess the include file with C preprocessor and with CPP preprocessor
|
||||
# - Pass the test if the preprocessor outputs are the same and whitespaces only (#define only header)
|
||||
# - Fail the test if the preprocessor outputs are the same (but with some code)
|
||||
# - If outputs different, continue with 2)
|
||||
# 2) Strip out all include directives to generate "temp.h"
|
||||
# 3) Preprocess the temp.h the same way in (1)
|
||||
# - Pass the test if the preprocessor outputs are the same and whitespaces only (#include only header)
|
||||
# - Fail the test if the preprocessor outputs are the same (but with some code)
|
||||
# - If outputs different, pass the test
|
||||
#
|
||||
preproc_one_header ${include_file} || ret=$?
|
||||
case $ret in
|
||||
1*) log_details "sdkconfig issue"; return 3 ;;
|
||||
2*) log_details "compile failed with #error"; return 0 ;;
|
||||
3*) log_details "another compile issue"; return 2 ;;
|
||||
4*) log_details "zero output"; return 0 ;;
|
||||
5*) log_details "the same non-zero out"; return 1 ;;
|
||||
[6-7]*) log_details "different and non-zero: check again without includes";
|
||||
# Removing includes to check if the header itself contains a code
|
||||
temp=$(mktemp)
|
||||
sed '/#include/d; /#error/d' ${include_file} > ${temp}
|
||||
case `preproc_one_header ${temp} || echo $?` in
|
||||
[1-3]*) log_details "ok, compile issue"; return 0 ;;
|
||||
4*) log_details "ok, zero output"; return 0 ;;
|
||||
5*) log_details "the same non-zero out"; return 1 ;;
|
||||
6*) log_details "different, non-zero, with extern \"C\""; return 0 ;;
|
||||
7*) log_details "different, non-zero, no extern \"C\""; return 1 ;;
|
||||
esac
|
||||
rm -f ${temp}
|
||||
esac
|
||||
}
|
||||
|
||||
# create_test_main <dir>
|
||||
#
|
||||
# Generates a compilation unit to fail if sdkconfig.h inclusion to be checked
|
||||
# Exports the full path to the file as TEST_MAIN_C
|
||||
function create_test_main
|
||||
{
|
||||
echo "#if defined(IDF_CHECK_SDKCONFIG_INCLUDED) && ! defined(IDF_SDKCONFIG_INCLUDED)
|
||||
#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED
|
||||
#endif" > ${1}/compile.c
|
||||
export TEST_MAIN_C=${1}/compile.c
|
||||
}
|
||||
|
||||
# get_include_dir_list <project_dir>
|
||||
#
|
||||
# Get the list of public include directories from compile_commands.json.
|
||||
# Stores the result in INCLUDE_DIRS variable.
|
||||
function get_include_dirs_list
|
||||
{
|
||||
local proj_dir=${1}
|
||||
local sdkinclude_dir=${proj_dir}/build/config
|
||||
pushd $proj_dir
|
||||
idf.py reconfigure
|
||||
|
||||
# Get the compiler command line from compile_commands.json
|
||||
# Poor man's replacement of `jq` in Python.
|
||||
extract_build_command_py='import json; print(json.load(open("build/compile_commands.json", "r"))[0]["command"])'
|
||||
build_command=$(python -c "${extract_build_command_py}")
|
||||
|
||||
# build the list of include directories, only considering the directories in "components/"
|
||||
include_dirs=""
|
||||
for token in ${build_command}; do
|
||||
if [[ "${token}" =~ ^-I.* ]]; then
|
||||
flags_include_dirs+=" $token"
|
||||
fi
|
||||
if [[ ! "${token}" =~ ^-I.*/components/.* ]]; then
|
||||
continue
|
||||
fi
|
||||
newline=$'\n'
|
||||
include_dirs="${include_dirs}${token##-I}${newline}"
|
||||
done
|
||||
|
||||
# Append a config value used to check if the sdkconfig included
|
||||
echo "#define IDF_SDKCONFIG_INCLUDED" >> ${sdkinclude_dir}/sdkconfig.h
|
||||
export INCLUDE_DIRS=${include_dirs}
|
||||
flags_include_dirs+=" -I ${sdkinclude_dir}"
|
||||
export FLAGS_INCLUDE_DIRS=${flags_include_dirs}
|
||||
}
|
||||
|
||||
# Utility convertion functions used for passing load sequences to processes (in form of one character)
|
||||
#
|
||||
# Returns ordinal number of supplied character
|
||||
ord() { LC_CTYPE=C printf %d "'$1"; }
|
||||
# Returns a character represented by ASCII code
|
||||
chr() { printf "\\$(printf "%o" $1)"; }
|
||||
|
||||
# check_headers_in_process <process_index> <list_of_includes> <list_of_failed_includes>
|
||||
#
|
||||
# Checks one include files at a time in a batch of headers referenced by indices from stdin
|
||||
# This function may be called from separate processes (used simple load balancing)
|
||||
# Periodically checks files from <list_of_includes>
|
||||
# Outputs failed files to <list_of_failed_includes>
|
||||
function check_headers_in_process
|
||||
{
|
||||
local proc="$1"
|
||||
local list_of_includes="$2"
|
||||
local list_of_failed_includes="$3"
|
||||
|
||||
list=(`cat ${list_of_includes}`)
|
||||
size=${#list[@]}
|
||||
let batch=$size/$JOBS_LOAD_CHAR_SIZE+1
|
||||
|
||||
# Reads one char (due to atomicity) from stdin
|
||||
# -ord(char) signifies an index to the list of files to be processed
|
||||
while IFS= read -n1 char; do
|
||||
batch_index=$(ord $char)
|
||||
let run=$batch_index-$JOBS_LOAD_CHAR_FROM || true
|
||||
for i in `seq 1 $batch`; do
|
||||
let file_index=$run*$batch+$i-1 || true
|
||||
if [ "$file_index" -lt "$size" ]; then
|
||||
# log_details "$proc: Processing $file_index (batch=$batch, run=$run)\n"
|
||||
include_file=${list[$file_index]}
|
||||
rel_inc_file=${include_file##${IDF_PATH}}
|
||||
printf "Processing ${rel_inc_file}"
|
||||
check_one_header ${include_file} && ret=0 || ret=$?
|
||||
case "$ret" in
|
||||
0*) printf " OK\n" ;;
|
||||
1*) printf " FAIL (missing c++ guard)\n" ;;
|
||||
2*) printf " FAIL (cannot compile)\n" ;;
|
||||
3*) printf " FAIL (missing include of sdkconfig.h)\n" ;;
|
||||
esac
|
||||
if [[ ! "$ret" == 0 ]]; then
|
||||
echo ${rel_inc_file} >> ${list_of_failed_includes}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done < $WORK_ITEM_PIPE;
|
||||
}
|
||||
|
||||
# check_include_dirs <include_dirs> <list_of_includes> <list_of_failed_includes>
|
||||
#
|
||||
# Check all header files in directories listed in include_dirs variable.
|
||||
# Places the list of header files where suspects missing C++ guard to <missing_guard_list_file> file.
|
||||
# Places the list of header files where compilation issues found to <compile_error_list_file> file.
|
||||
# Calls check_one_header function to check a single header file.
|
||||
function check_include_dirs
|
||||
{
|
||||
local include_dirs="$1"
|
||||
local list_of_includes="$2"
|
||||
local list_of_failed_includes="$3"
|
||||
local include_dir_maxdepth=${INCLUDE_DIR_MAXDEPTH:-}
|
||||
local grep_expr=$(mktemp)
|
||||
|
||||
rm -f ${list_of_includes} ${list_of_failed_includes}
|
||||
touch ${list_of_includes} ${list_of_failed_includes}
|
||||
|
||||
if [ -z "${PROCESS_ALL_INCLUDES:-}" ]; then
|
||||
# generate grep expr: 1) strip comments, 2) generate '<exclude1>\|<exclude2>\|...'
|
||||
exclusions=`cat $EXCLUSIONS_LIST | sed '/^#/d'`
|
||||
echo $exclusions | sed 's/ /\\|/g' > ${grep_expr}
|
||||
fi;
|
||||
|
||||
for include_dir in ${include_dirs}; do
|
||||
echo "Processing include directory: ${include_dir}"
|
||||
include_files=$(find ${include_dir} ${include_dir_maxdepth} -name '*.h')
|
||||
for include_file in ${include_files}; do
|
||||
if [ -z "${PROCESS_ALL_INCLUDES:-}" ] && echo "$include_file" | grep -q -f ${grep_expr}; then
|
||||
continue
|
||||
fi
|
||||
echo ${include_file} >> ${list_of_includes}
|
||||
done
|
||||
done
|
||||
rm -f ${grep_expr}
|
||||
}
|
||||
|
||||
# filter_exclusions <exclusions_list_file> <filtered_list_file>
|
||||
#
|
||||
# Takes the list of files from stdin, and removes all files which
|
||||
# are listed in <exclusions_list_file> file.
|
||||
# Saves the final (filtered) list in <filtered_list_file> file.
|
||||
# The exclusions list file may contain comments and empty lines.
|
||||
function filter_exclusions
|
||||
{
|
||||
local exclusions_list_file=$1
|
||||
local filtered_list_file=$2
|
||||
local grep_expr=$(mktemp)
|
||||
|
||||
# generate grep expr: 1) strip comments, 2) generate '<exclude1>\|<exclude2>\|...'
|
||||
exclusions=`cat $exclusions_list_file | sed '/^#/d'`
|
||||
echo $exclusions | sed 's/ /\\|/g' > ${grep_expr}
|
||||
|
||||
# pass the expression as file (input goes from stdin)
|
||||
grep -vf ${grep_expr} > ${filtered_list_file} || true
|
||||
rm -f ${grep_expr}
|
||||
}
|
||||
|
||||
##################### main logic starts here #####################
|
||||
|
||||
EXCLUSIONS_LIST=${IDF_PATH}/tools/ci/check_public_headers_exceptions.txt
|
||||
FAILED_INCLUDE_LIST=failed.txt
|
||||
TOTAL_INCLUDE_LIST=include_list.txt
|
||||
FILTERED_LIST=filtered.txt
|
||||
LINE=---------------------------------------------------------------
|
||||
CC=${CC:-gcc}
|
||||
CXX=${CXX:-g++}
|
||||
TEST_PROJECT_DIR=${IDF_PATH}/examples/get-started/blink
|
||||
MULTI_JOBS="1"
|
||||
# jobs, load balancing vars, temp outputs
|
||||
JOBS_LOAD_CHAR_FROM=64
|
||||
JOBS_LOAD_CHAR_TO=90
|
||||
let JOBS_LOAD_CHAR_SIZE=$JOBS_LOAD_CHAR_TO-$JOBS_LOAD_CHAR_FROM+1
|
||||
TEMP_OUT_FILE=/tmp/proc_out.txt.
|
||||
WORK_ITEM_PIPE=/tmp/work_item_pipe
|
||||
|
||||
# Process command line arguments
|
||||
for i in $@; do
|
||||
if [[ "$i" =~ "--verbose" ]]; then
|
||||
echo "$0: Enabling verbose output"
|
||||
VERBOSE="1"
|
||||
fi
|
||||
if [[ "$i" =~ "--process-all-files" ]]; then
|
||||
echo "$0: Processing all files"
|
||||
PROCESS_ALL_INCLUDES="1"
|
||||
fi
|
||||
if [[ "$i" =~ --include-maxdepth=* ]]; then
|
||||
INCLUDE_DIR_MAXDEPTH="-maxdepth ${i##--include-maxdepth=}"
|
||||
echo "$0: Searching for includes with: $INCLUDE_DIR_MAXDEPTH"
|
||||
fi
|
||||
if [[ "$i" =~ --jobs=* ]]; then
|
||||
MULTI_JOBS="${i##--jobs=}"
|
||||
echo "$0: Checking headers within multiple jobs: $MULTI_JOBS"
|
||||
fi
|
||||
done
|
||||
|
||||
[[ -p $WORK_ITEM_PIPE ]] || mkfifo $WORK_ITEM_PIPE
|
||||
|
||||
echo $LINE
|
||||
# Get include directories
|
||||
get_include_dirs_list ${TEST_PROJECT_DIR}
|
||||
# Generate the main file to compiled while preincluding a header under test
|
||||
create_test_main ${TEST_PROJECT_DIR}/build
|
||||
# Get the list of include files to be processed
|
||||
check_include_dirs "${INCLUDE_DIRS}" ${TOTAL_INCLUDE_LIST} ${FAILED_INCLUDE_LIST}
|
||||
|
||||
# Launching separate processes (if $MULTI_JOBS defined) to check includes
|
||||
#
|
||||
# poor man's approach to load balancing:
|
||||
# 1) sending work items (chars A-Z for atomicity) to a named pipe
|
||||
# 2) spawning nr of processes to pick the work items from the pipe
|
||||
# 3) every process reads one char at a time to analyze a batch of includes (batch=total_includes/nr_of_work_items)
|
||||
# 4) waiting till all processes finish
|
||||
|
||||
# running the jobs and marking the pid
|
||||
for ((i=1; i<${MULTI_JOBS}; i++)); do
|
||||
check_headers_in_process $i ${TOTAL_INCLUDE_LIST} ${FAILED_INCLUDE_LIST} > ${TEMP_OUT_FILE}${i} &
|
||||
pids[${i}]=$!
|
||||
done
|
||||
# one job outputs directly to stdout (to show the script is alive)
|
||||
check_headers_in_process 0 ${TOTAL_INCLUDE_LIST} ${FAILED_INCLUDE_LIST} &
|
||||
pid0=$!
|
||||
|
||||
# sending sequence of work items
|
||||
for i in `seq $JOBS_LOAD_CHAR_FROM $JOBS_LOAD_CHAR_TO`; do
|
||||
printf $(chr $i )
|
||||
done > $WORK_ITEM_PIPE
|
||||
|
||||
# waiting for the processes to finish and print their outputs
|
||||
for ((i=1; i<${MULTI_JOBS}; i++)); do
|
||||
log_details "\nwaiting for process nr: $i\n"
|
||||
wait ${pids[$i]} || true
|
||||
cat ${TEMP_OUT_FILE}${i}
|
||||
rm -f ${TEMP_OUT_FILE}${i}
|
||||
done
|
||||
# and the last one
|
||||
wait $pid0 || true
|
||||
rm -f $WORK_ITEM_PIPE
|
||||
|
||||
# Filter out the known issues and report results
|
||||
cat ${FAILED_INCLUDE_LIST} | filter_exclusions ${EXCLUSIONS_LIST} ${FILTERED_LIST}
|
||||
echo $LINE
|
||||
|
||||
if [ -s ${FILTERED_LIST} ]; then
|
||||
echo "The following files failed:"
|
||||
echo ""
|
||||
cat ${FILTERED_LIST}
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "No errors found"
|
||||
|
|
@ -1,23 +1,23 @@
|
|||
### General ignore list
|
||||
#
|
||||
components/xtensa/include/xtensa/*
|
||||
components/xtensa/include/*
|
||||
components/xtensa/esp32/include/xtensa/config/*
|
||||
components/xtensa/include/xtensa/
|
||||
components/xtensa/include/
|
||||
components/xtensa/esp32/include/xtensa/config/
|
||||
|
||||
components/newlib/platform_include/*
|
||||
components/newlib/platform_include/
|
||||
|
||||
components/freertos/include/freertos/*
|
||||
components/freertos/xtensa/include/freertos/*
|
||||
components/freertos/include/freertos/
|
||||
components/freertos/xtensa/include/freertos/
|
||||
|
||||
|
||||
components/log/include/esp_log_internal.h
|
||||
|
||||
components/soc/include/hal/*
|
||||
components/soc/include/soc/*
|
||||
components/soc/include/hal/
|
||||
components/soc/include/soc/
|
||||
|
||||
components/esp_rom/include/esp32s2/rom/rsa_pss.h
|
||||
|
||||
components/esp_common/include/esp_private/*
|
||||
components/esp_common/include/esp_private/
|
||||
|
||||
components/esp32/include/esp32/brownout.h
|
||||
components/esp32/include/esp32/spiram.h
|
||||
|
|
@ -46,28 +46,28 @@ components/spi_flash/include/spi_flash_chip_generic.h
|
|||
|
||||
components/bootloader_support/include/esp_app_format.h
|
||||
|
||||
components/wpa_supplicant/include/*
|
||||
components/wpa_supplicant/port/*
|
||||
components/wpa_supplicant/include/
|
||||
components/wpa_supplicant/port/
|
||||
|
||||
components/mbedtls/port/include/*
|
||||
components/mbedtls/mbedtls/include/mbedtls/*
|
||||
components/mbedtls/port/include/
|
||||
components/mbedtls/mbedtls/include/mbedtls/
|
||||
|
||||
components/espcoredump/include/esp_core_dump.h
|
||||
|
||||
components/coap/*
|
||||
components/nghttp/*
|
||||
components/cbor/*
|
||||
components/coap/
|
||||
components/nghttp/
|
||||
components/cbor/
|
||||
|
||||
components/esp-tls/private_include/*
|
||||
components/esp-tls/private_include/
|
||||
|
||||
components/protobuf-c/*
|
||||
components/protobuf-c/
|
||||
|
||||
components/mdns/include/mdns_console.h
|
||||
|
||||
components/esp_serial_slave_link/include/esp_serial_slave_link/essl.h
|
||||
components/esp_serial_slave_link/include/esp_serial_slave_link/essl_sdio.h
|
||||
|
||||
components/expat/*
|
||||
components/expat/
|
||||
|
||||
components/fatfs/vfs/vfs_fat_internal.h
|
||||
components/fatfs/src/ffconf.h
|
||||
|
|
@ -78,9 +78,9 @@ components/freemodbus/common/include/esp_modbus_slave.h
|
|||
|
||||
components/idf_test/include/idf_performance.h
|
||||
|
||||
components/json/cJSON/*
|
||||
components/json/cJSON/
|
||||
|
||||
components/libsodium/*
|
||||
components/libsodium/
|
||||
|
||||
components/spiffs/include/spiffs_config.h
|
||||
|
||||
|
|
@ -100,3 +100,53 @@ components/heap/include/esp_heap_task_info.h
|
|||
components/esp_wifi/include/esp_private/wifi_os_adapter.h
|
||||
components/asio/port/include/esp_exception.h
|
||||
components/esp_common/include/esp_compiler.h
|
||||
|
||||
### To be fixed: headers that rely on implicit inclusion
|
||||
#
|
||||
components/lwip/lwip/src/include/lwip/prot/nd6.h
|
||||
components/lwip/port/esp32/include/netif/dhcp_state.h
|
||||
components/soc/src/esp32/rtc_clk_common.h
|
||||
components/soc/src/esp32/i2c_rtc_clk.h
|
||||
components/soc/src/esp32/include/hal/cpu_ll.h
|
||||
components/soc/src/esp32/include/hal/timer_ll.h
|
||||
components/soc/src/esp32/include/hal/cpu_ll.h
|
||||
components/soc/src/esp32/include/hal/timer_ll.h
|
||||
components/esp_rom/include/esp32/rom/sha.h
|
||||
components/esp_rom/include/esp32/rom/secure_boot.h
|
||||
components/esp_rom/include/esp32s2/rom/spi_flash.h
|
||||
components/esp_rom/include/esp32s2/rom/cache.h
|
||||
components/esp_rom/include/esp32s2/rom/secure_boot.h
|
||||
components/esp_rom/include/esp32s2/rom/opi_flash.h
|
||||
components/esp_rom/include/esp32s2/rom/efuse.h
|
||||
components/esp_common/include/esp_freertos_hooks.h
|
||||
components/esp32/include/esp32/dport_access.h
|
||||
components/esp32/include/rom/sha.h
|
||||
components/esp32/include/rom/secure_boot.h
|
||||
components/driver/esp32/include/touch_sensor.h
|
||||
components/esp_ringbuf/include/freertos/ringbuf.h
|
||||
components/efuse/esp32/include/esp_efuse_table.h
|
||||
components/esp_wifi/include/esp_wifi_crypto_types.h
|
||||
components/esp_wifi/include/esp_coexist_internal.h
|
||||
components/esp_wifi/include/esp_wifi_netif.h
|
||||
components/esp_wifi/include/smartconfig_ack.h
|
||||
components/esp_wifi/include/esp_wifi_default.h
|
||||
components/esp_wifi/include/esp_coexist_adapter.h
|
||||
components/esp_event/include/esp_event_base.h
|
||||
components/esp_netif/include/esp_netif_sta_list.h
|
||||
components/esp_netif/include/esp_netif_defaults.h
|
||||
components/esp_netif/include/esp_netif_net_stack.h
|
||||
components/esp_netif/include/esp_netif_types.h
|
||||
components/esp_netif/include/esp_netif_ip_addr.h
|
||||
components/esp_netif/include/esp_netif_ppp.h
|
||||
components/tcpip_adapter/include/tcpip_adapter_compatible/tcpip_adapter_compat.h
|
||||
components/bootloader_support/include/bootloader_util.h
|
||||
components/tcpip_adapter/include/tcpip_adapter_types.h
|
||||
components/console/linenoise/linenoise.h
|
||||
components/protocomm/include/transports/protocomm_httpd.h
|
||||
components/fatfs/src/diskio.h
|
||||
components/fatfs/diskio/diskio_sdmmc.h
|
||||
components/openssl/include/openssl/ssl.h
|
||||
components/ulp/include/ulp_common.h
|
||||
components/lwip/include/apps/sntp/sntp.h
|
||||
components/mbedtls/esp_crt_bundle/include/esp_crt_bundle.h
|
||||
components/wifi_provisioning/include/wifi_provisioning/scheme_softap.h
|
||||
|
|
|
|||
|
|
@ -187,5 +187,5 @@ check_public_headers:
|
|||
- $BOT_LABEL_BUILD
|
||||
- $BOT_LABEL_REGULAR_TEST
|
||||
script:
|
||||
- CC=xtensa-esp32-elf-gcc CXX=xtensa-esp32-elf-g++ tools/ci/check_public_headers.sh --verbose --process-all-files --jobs=4
|
||||
- python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue