From 1358734b3f1990816c94d2d12ccfee9caf088095 Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Mon, 8 Jun 2020 16:37:28 +0800 Subject: [PATCH 1/3] Fixes issues with gen_crt_bundle.py for unicode chars gen_crt_bundle.py could fail to parse the certificates if it contained a non-ascii character. --- .../mbedtls/esp_crt_bundle/gen_crt_bundle.py | 7 +++--- .../test_gen_crt_bundle/non_ascii_crt.pem | 23 +++++++++++++++++++ .../test_gen_crt_bundle.py | 7 ++++++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/non_ascii_crt.pem diff --git a/components/mbedtls/esp_crt_bundle/gen_crt_bundle.py b/components/mbedtls/esp_crt_bundle/gen_crt_bundle.py index 94f31e433..4f558fb68 100755 --- a/components/mbedtls/esp_crt_bundle/gen_crt_bundle.py +++ b/components/mbedtls/esp_crt_bundle/gen_crt_bundle.py @@ -30,6 +30,7 @@ import struct import argparse import csv import re +from io import open try: from cryptography import x509 @@ -80,7 +81,7 @@ class CertificateBundle: try: if file_path.endswith('.pem'): status("Parsing certificates from %s" % file_path) - with open(file_path, 'r') as f: + with open(file_path, 'r', encoding='utf-8') as f: crt_str = f.read() self.add_from_pem(crt_str) return True @@ -153,7 +154,7 @@ class CertificateBundle: def add_with_filter(self, crts_path, filter_path): filter_set = set() - with open(filter_path, 'r') as f: + with open(filter_path, 'r', encoding='utf-8') as f: csv_reader = csv.reader(f, delimiter=',') # Skip header @@ -163,7 +164,7 @@ class CertificateBundle: status("Parsing certificates from %s" % crts_path) crt_str = [] - with open(crts_path, 'r') as f: + with open(crts_path, 'r', encoding='utf-8') as f: crt_str = f.read() # Split all certs into a list of (name, certificate string) tuples diff --git a/components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/non_ascii_crt.pem b/components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/non_ascii_crt.pem new file mode 100644 index 000000000..10ad8a1ca --- /dev/null +++ b/components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/non_ascii_crt.pem @@ -0,0 +1,23 @@ +NetLock Arany (Class Gold) Főtanúsítvány +======================================== +-----BEGIN CERTIFICATE----- +MIIEFTCCAv2gAwIBAgIGSUEs5AAQMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYDVQQGEwJIVTERMA8G +A1UEBwwIQnVkYXBlc3QxFTATBgNVBAoMDE5ldExvY2sgS2Z0LjE3MDUGA1UECwwuVGFuw7pzw610 +dsOhbnlraWFkw7NrIChDZXJ0aWZpY2F0aW9uIFNlcnZpY2VzKTE1MDMGA1UEAwwsTmV0TG9jayBB +cmFueSAoQ2xhc3MgR29sZCkgRsWRdGFuw7pzw610dsOhbnkwHhcNMDgxMjExMTUwODIxWhcNMjgx +MjA2MTUwODIxWjCBpzELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRUwEwYDVQQKDAxO +ZXRMb2NrIEtmdC4xNzA1BgNVBAsMLlRhbsO6c8OtdHbDoW55a2lhZMOzayAoQ2VydGlmaWNhdGlv +biBTZXJ2aWNlcykxNTAzBgNVBAMMLE5ldExvY2sgQXJhbnkgKENsYXNzIEdvbGQpIEbFkXRhbsO6 +c8OtdHbDoW55MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxCRec75LbRTDofTjl5Bu +0jBFHjzuZ9lk4BqKf8owyoPjIMHj9DrTlF8afFttvzBPhCf2nx9JvMaZCpDyD/V/Q4Q3Y1GLeqVw +/HpYzY6b7cNGbIRwXdrzAZAj/E4wqX7hJ2Pn7WQ8oLjJM2P+FpD/sLj916jAwJRDC7bVWaaeVtAk +H3B5r9s5VA1lddkVQZQBr17s9o3x/61k/iCa11zr/qYfCGSji3ZVrR47KGAuhyXoqq8fxmRGILdw +fzzeSNuWU7c5d+Qa4scWhHaXWy+7GRWF+GmF9ZmnqfI0p6m2pgP8b4Y9VHx2BJtr+UBdADTHLpl1 +neWIA6pN+APSQnbAGwIDAKiLo0UwQzASBgNVHRMBAf8ECDAGAQH/AgEEMA4GA1UdDwEB/wQEAwIB +BjAdBgNVHQ4EFgQUzPpnk/C2uNClwB7zU/2MU9+D15YwDQYJKoZIhvcNAQELBQADggEBAKt/7hwW +qZw8UQCgwBEIBaeZ5m8BiFRhbvG5GK1Krf6BQCOUL/t1fC8oS2IkgYIL9WHxHG64YTjrgfpioTta +YtOUZcTh5m2C+C8lcLIhJsFyUR+MLMOEkMNaj7rP9KdlpeuY0fsFskZ1FSNqb4VjMIDw1Z4fKRzC +bLBQWV2QWzuoDTDPv31/zvGdg73JRm4gpvlhUbohL3u+pRVjodSVh/GeufOJ8z2FuLjbvrW5Kfna +NwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2XjG4Kvte9nHfRCaexOYNkbQu +dZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E= +-----END CERTIFICATE----- diff --git a/components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py b/components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py index b502cea92..167aa6384 100755 --- a/components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py +++ b/components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py @@ -22,6 +22,7 @@ pem_test_file = 'entrust.pem' verified_der_bundle = 'baltimore_crt_bundle' verified_pem_bundle = 'entrust_crt_bundle' invalid_test_file = 'invalid_crt.pem' +non_ascii_file = 'non_ascii_crt.pem' ca_crts_all_file = 'cacrt_all.pem' @@ -72,6 +73,12 @@ class GenCrtBundleTests(Py23TestCase): with self.assertRaisesRegex(gen_crt_bundle.InputError, "No certificate found"): bundle.add_from_pem("") + def test_non_ascii_crt_input(self): + bundle = gen_crt_bundle.CertificateBundle() + + bundle.add_from_file(test_crts_path + non_ascii_file) + self.assertTrue(len(bundle.certificates)) + if __name__ == "__main__": unittest.main() From 1531f2db2aa97d49ea82c59a258472aca8ad4129 Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Tue, 9 Jun 2020 10:12:15 +0800 Subject: [PATCH 2/3] ci: fix check_public_headers.py failure with python3 due to non ascii chars check_public_headers.py would fail when reading non-ascii chars on systems where the default encoding was ascii. Also fixes error handling issues, as any uncaught exeception would cause the program to run indefinitely. --- tools/ci/check_public_headers.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/ci/check_public_headers.py b/tools/ci/check_public_headers.py index f936296c4..af934acd9 100644 --- a/tools/ci/check_public_headers.py +++ b/tools/ci/check_public_headers.py @@ -28,6 +28,7 @@ import argparse import queue from threading import Thread, Event import tempfile +from io import open class HeaderFailed(Exception): @@ -60,7 +61,7 @@ class HeaderFailedContainsCode(HeaderFailed): def exec_cmd_to_temp_file(what, suffix=""): out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) rc, out, err = exec_cmd(what, out_file) - with open(out_file.name, "r") as f: + with open(out_file.name, "r", encoding='utf-8') as f: out = f.read() return rc, out, err, out_file.name @@ -129,6 +130,11 @@ class PublicHeaderChecker: self.check_one_header(task, num) except HeaderFailed as e: self.failed_queue.put("{}: Failed! {}".format(task, e)) + except Exception as e: + # Makes sure any unexpected exceptions causes the program to terminate + self.failed_queue.put("{}: Failed! {}".format(task, e)) + self.terminate.set() + raise def get_failed(self): return list(self.failed_queue.queue) @@ -166,6 +172,7 @@ class PublicHeaderChecker: raise HeaderFailedCppGuardMissing() else: self.compile_one_header(header) + temp_header = None try: _, _, _, temp_header = exec_cmd_to_temp_file(["sed", "/#include/d; /#error/d", header], suffix=".h") res = self.preprocess_one_header(temp_header, num, ignore_sdkconfig_issue=True) @@ -174,7 +181,8 @@ class PublicHeaderChecker: elif res == self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED: raise HeaderFailedCppGuardMissing() finally: - os.unlink(temp_header) + if temp_header: + os.unlink(temp_header) def compile_one_header(self, header): rc, out, err = exec_cmd([self.gcc, "-S", "-o-", "-include", header, self.main_c] + self.include_dir_flags) @@ -238,7 +246,7 @@ class PublicHeaderChecker: project_dir = os.path.join(idf_path, "examples", "get-started", "blink") subprocess.check_call(["idf.py", "reconfigure"], cwd=project_dir) build_commands_json = os.path.join(project_dir, "build", "compile_commands.json") - with open(build_commands_json, "r") as f: + with open(build_commands_json, "r", encoding='utf-8') as f: build_command = json.load(f)[0]["command"].split() include_dir_flags = [] include_dirs = [] @@ -303,7 +311,7 @@ def check_all_headers(): # process excluded files and dirs exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file) - with open(exclude_file, "r") as f: + with open(exclude_file, "r", encoding='utf-8') as f: lines = [line.rstrip() for line in f] ignore_files = [] ignore_dirs = [] From 1d1e4497baf32d7ecd511ff01ec9723ba6380854 Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Tue, 9 Jun 2020 11:35:08 +0800 Subject: [PATCH 3/3] ci: fix find_build_apps.py failure with py3 due to non ascii chars find_build_apps.py would fail when reading non-ascii chars on systems where the default encoding was ascii. --- tools/find_build_apps/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index 3faf5341b..ddc16093a 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -8,6 +8,7 @@ from collections import namedtuple import logging import json import typing +from io import open DEFAULT_TARGET = "esp32" @@ -320,7 +321,7 @@ class BuildSystem(object): readme_path = get_md_or_rst(os.path.dirname(app_path)) if not readme_path: return None - with open(readme_path, "r") as readme_file: + with open(readme_path, "r", encoding='utf8') as readme_file: return readme_file.read() @staticmethod