From 9daf51e6be6991d6c952d587f47023eaef67ece4 Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Wed, 10 Oct 2018 13:19:31 +0200 Subject: [PATCH] Fix Python 3 compatibility issues --- docs/conf_common.py | 2 ++ docs/docs_common.mk | 2 +- docs/gen-version-specific-includes.py | 4 ++-- .../asio/chat_client/asio_chat_client_test.py | 11 +++++----- .../asio/chat_server/asio_chat_server_test.py | 2 +- .../tcp_echo_server/asio_tcp_server_test.py | 4 ++-- .../udp_echo_server/asio_udp_server_test.py | 4 ++-- tools/ci/check_artifacts_expire_time.py | 2 +- tools/tiny-test-fw/Utility/Attenuator.py | 7 ++++--- tools/tiny-test-fw/Utility/PowerControl.py | 20 +++++++++---------- .../unit-test-app/tools/CreateSectionTable.py | 12 +++++------ tools/unit-test-app/tools/UnitTestParser.py | 6 +++--- 12 files changed, 40 insertions(+), 36 deletions(-) diff --git a/docs/conf_common.py b/docs/conf_common.py index 0eafe710d..8ffd0b3ec 100644 --- a/docs/conf_common.py +++ b/docs/conf_common.py @@ -27,6 +27,8 @@ import shlex from local_util import run_cmd_get_output, copy_if_modified +# build_docs on the CI server sometimes fails under Python3. This is a workaround: +sys.setrecursionlimit(3500) try: builddir = os.environ['BUILDDIR'] diff --git a/docs/docs_common.mk b/docs/docs_common.mk index d4d623ec8..7a77b5729 100644 --- a/docs/docs_common.mk +++ b/docs/docs_common.mk @@ -17,7 +17,7 @@ SPHINXOPTS = # note: this is changed from sphinx-build so it depends on default python interpreter, not on /bin/sphinx-build # (which will be the most recently installed version of sphinx and may not match) -SPHINXBUILD = python2 -m sphinx +SPHINXBUILD = python -m sphinx PAPER = BUILDDIR = _build diff --git a/docs/gen-version-specific-includes.py b/docs/gen-version-specific-includes.py index 57916e42d..fd2264073 100755 --- a/docs/gen-version-specific-includes.py +++ b/docs/gen-version-specific-includes.py @@ -170,9 +170,9 @@ def get_version(): # No tag, look for a branch refs = subprocess.check_output(["git", "for-each-ref", "--points-at", "HEAD", "--format", "%(refname)"]) print("refs:\n%s" % refs) - refs = refs.split("\n") + refs = refs.split(b"\n") # Note: this looks for branches in 'origin' because GitLab CI doesn't check out a local branch - branches = [ r.replace("refs/remotes/origin/","").strip() for r in refs if r.startswith("refs/remotes/origin/") ] + branches = [ r.replace(b"refs/remotes/origin/",b"").strip() for r in refs if r.startswith(b"refs/remotes/origin/") ] if len(branches) == 0: # last resort, return the commit (may happen on Gitlab CI sometimes, unclear why) return (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).strip(), "commit", False) diff --git a/examples/protocols/asio/chat_client/asio_chat_client_test.py b/examples/protocols/asio/chat_client/asio_chat_client_test.py index c8b9aacc6..37018ed37 100644 --- a/examples/protocols/asio/chat_client/asio_chat_client_test.py +++ b/examples/protocols/asio/chat_client/asio_chat_client_test.py @@ -19,8 +19,8 @@ import IDF global g_client_response; global g_msg_to_client; -g_client_response = "" -g_msg_to_client = " 3XYZ" +g_client_response = b"" +g_msg_to_client = b" 3XYZ" def get_my_ip(): s1 = socket(AF_INET, SOCK_DGRAM) @@ -81,11 +81,12 @@ def test_examples_protocol_asio_chat_client(env, extra_data): # 3. send host's IP to the client i.e. the `dut1` dut1.write(host_ip) # 4. client `dut1` should receive a message - dut1.expect(g_msg_to_client[4:]) # Strip out the front 4 bytes of message len (see chat_message protocol) + dut1.expect(g_msg_to_client[4:].decode()) # Strip out the front 4 bytes of message len (see chat_message protocol) # 5. write test message from `dut1` chat_client to the server dut1.write(test_msg) - while g_client_response == "": + while len(g_client_response) == 0: time.sleep(1) + g_client_response = g_client_response.decode() print(g_client_response) # 6. evaluate host_server received this message if (g_client_response[4:7] == test_msg): @@ -93,7 +94,7 @@ def test_examples_protocol_asio_chat_client(env, extra_data): pass else: print("Failure!") - raise ValueError('Wrong data received from asi tcp server: {} (expected:{})'.format(g_client_response, test_msg)) + raise ValueError('Wrong data received from asi tcp server: {} (expected:{})'.format(g_client_response[4:7], test_msg)) thread1.join() if __name__ == '__main__': diff --git a/examples/protocols/asio/chat_server/asio_chat_server_test.py b/examples/protocols/asio/chat_server/asio_chat_server_test.py index ac90a9261..7f25dc512 100644 --- a/examples/protocols/asio/chat_server/asio_chat_server_test.py +++ b/examples/protocols/asio/chat_server/asio_chat_server_test.py @@ -27,7 +27,7 @@ def test_examples_protocol_asio_chat_server(env, extra_data): 3. Test connects to server and sends a test message 4. Test evaluates received test message from server """ - test_msg=" 4ABC\n" + test_msg=b" 4ABC\n" dut1 = env.get_dut("chat_server", "examples/protocols/asio/chat_server") # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "asio_chat_server.bin") diff --git a/examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py b/examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py index ad93a5596..840060d4c 100644 --- a/examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py +++ b/examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py @@ -28,7 +28,7 @@ def test_examples_protocol_asio_tcp_server(env, extra_data): 4. Test evaluates received test message from server 5. Test evaluates received test message on server stdout """ - test_msg="echo message from client to server" + test_msg=b"echo message from client to server" dut1 = env.get_dut("tcp_echo_server", "examples/protocols/asio/tcp_echo_server") # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "asio_tcp_echo_server.bin") @@ -53,7 +53,7 @@ def test_examples_protocol_asio_tcp_server(env, extra_data): print("Failure!") raise ValueError('Wrong data received from asi tcp server: {} (expoected:{})'.format(data, test_msg)) # 5. check the client message appears also on server terminal - dut1.expect(test_msg) + dut1.expect(test_msg.decode()) if __name__ == '__main__': diff --git a/examples/protocols/asio/udp_echo_server/asio_udp_server_test.py b/examples/protocols/asio/udp_echo_server/asio_udp_server_test.py index 44a577369..5487700ff 100644 --- a/examples/protocols/asio/udp_echo_server/asio_udp_server_test.py +++ b/examples/protocols/asio/udp_echo_server/asio_udp_server_test.py @@ -28,7 +28,7 @@ def test_examples_protocol_asio_udp_server(env, extra_data): 4. Test evaluates received test message from server 5. Test evaluates received test message on server stdout """ - test_msg="echo message from client to server" + test_msg=b"echo message from client to server" dut1 = env.get_dut("udp_echo_server", "examples/protocols/asio/udp_echo_server") # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "asio_udp_echo_server.bin") @@ -53,7 +53,7 @@ def test_examples_protocol_asio_udp_server(env, extra_data): print("Failure!") raise ValueError('Wrong data received from asi udp server: {} (expoected:{})'.format(data, test_msg)) # 5. check the client message appears also on server terminal - dut1.expect(test_msg) + dut1.expect(test_msg.decode()) if __name__ == '__main__': test_examples_protocol_asio_udp_server() diff --git a/tools/ci/check_artifacts_expire_time.py b/tools/ci/check_artifacts_expire_time.py index 471e59a39..3eb06c177 100644 --- a/tools/ci/check_artifacts_expire_time.py +++ b/tools/ci/check_artifacts_expire_time.py @@ -23,7 +23,7 @@ def check_artifacts_expire_time(): print("expire time for jobs:") - job_names = config.keys() + job_names = list(config.keys()) job_names.sort() for job_name in job_names: diff --git a/tools/tiny-test-fw/Utility/Attenuator.py b/tools/tiny-test-fw/Utility/Attenuator.py index 451c48811..7a188ff9b 100644 --- a/tools/tiny-test-fw/Utility/Attenuator.py +++ b/tools/tiny-test-fw/Utility/Attenuator.py @@ -20,6 +20,7 @@ This file provide method to control programmable attenuator. import time import serial +import codecs def set_att(port, att, att_fix=False): @@ -51,11 +52,11 @@ def set_att(port, att, att_fix=False): cmd_hex = "7e7e10{:02x}{:x}".format(att_t, 0x10 + att_t) exp_res_hex = "7e7e20{:02x}00{:x}".format(att_t, 0x20 + att_t) - cmd = cmd_hex.decode("hex") - exp_res = exp_res_hex.decode("hex") + cmd = codecs.decode(cmd_hex, "hex") + exp_res = codecs.decode(exp_res_hex, "hex") serial_port.write(cmd) - res = "" + res = b"" for i in range(5): res += serial_port.read(20) diff --git a/tools/tiny-test-fw/Utility/PowerControl.py b/tools/tiny-test-fw/Utility/PowerControl.py index 5b7df4c4f..a55abd5ac 100644 --- a/tools/tiny-test-fw/Utility/PowerControl.py +++ b/tools/tiny-test-fw/Utility/PowerControl.py @@ -27,15 +27,15 @@ class Control(object): @classmethod def apc_telnet_make_choice(cls, telnet, choice): """ select a choice """ - telnet.read_until("Event Log") - telnet.read_until(">") - telnet.write(choice + "\r\n") + telnet.read_until(b"Event Log") + telnet.read_until(b">") + telnet.write(choice.encode() + b"\r\n") @classmethod def apc_telnet_common_action(cls, telnet, check_str, action): """ wait until a pattern and then write a line """ - telnet.read_until(check_str) - telnet.write(action + "\r\n") + telnet.read_until(check_str.encode()) + telnet.write(action.encode() + b"\r\n") @classmethod def control(cls, apc_ip, control_dict): @@ -83,13 +83,13 @@ class Control(object): cls.apc_telnet_make_choice(tn, "\033") # exit to main menu and logout - tn.write("\033\r\n") - tn.write("\033\r\n") - tn.write("\033\r\n") - tn.write("4\r\n") + tn.write(b"\033\r\n") + tn.write(b"\033\r\n") + tn.write(b"\033\r\n") + tn.write(b"4\r\n") @classmethod def control_rest(cls, apc_ip, outlet, action): - outlet_list = range(1, 9) + outlet_list = list(range(1, 9)) # has to be a list if we want to remove from it under Python 3 outlet_list.remove(outlet) cls.control(apc_ip, dict.fromkeys(outlet_list, action)) diff --git a/tools/unit-test-app/tools/CreateSectionTable.py b/tools/unit-test-app/tools/CreateSectionTable.py index a9379cf04..ca5114c6b 100644 --- a/tools/unit-test-app/tools/CreateSectionTable.py +++ b/tools/unit-test-app/tools/CreateSectionTable.py @@ -6,8 +6,8 @@ class Section(object): """ One Section of section table. contains info about section name, address and raw data """ - SECTION_START_PATTERN = re.compile("Contents of section (.+?):") - DATA_PATTERN = re.compile("([0-9a-f]{4,8})") + SECTION_START_PATTERN = re.compile(b"Contents of section (.+?):") + DATA_PATTERN = re.compile(b"([0-9a-f]{4,8})") def __init__(self, name, start_address, data): self.name = name @@ -52,7 +52,7 @@ class Section(object): start_address = 0 # first find start line for i, line in enumerate(raw_data): - if "Contents of section " in line: # do strcmp first to speed up + if b"Contents of section " in line: # do strcmp first to speed up match = cls.SECTION_START_PATTERN.search(line) if match is not None: name = match.group(1) @@ -60,11 +60,11 @@ class Section(object): break else: # do some error handling - raw_data = [""] # add a dummy first data line + raw_data = [b""] # add a dummy first data line def process_data_line(line_to_process): # first remove the ascii part - hex_part = line_to_process.split(" ")[0] + hex_part = line_to_process.split(b" ")[0] # process rest part data_list = cls.DATA_PATTERN.findall(hex_part) try: @@ -74,7 +74,7 @@ class Section(object): def hex_to_str(hex_data): if len(hex_data) % 2 == 1: - hex_data = "0" + hex_data # append zero at the beginning + hex_data = b"0" + hex_data # append zero at the beginning _length = len(hex_data) return "".join([chr(int(hex_data[_i:_i + 2], base=16)) for _i in range(0, _length, 2)]) diff --git a/tools/unit-test-app/tools/UnitTestParser.py b/tools/unit-test-app/tools/UnitTestParser.py index 5b724e518..ec16583eb 100644 --- a/tools/unit-test-app/tools/UnitTestParser.py +++ b/tools/unit-test-app/tools/UnitTestParser.py @@ -68,7 +68,7 @@ class Parser(object): table = CreateSectionTable.SectionTable("section_table.tmp") tags = self.parse_tags(os.path.join(config_output_folder, self.SDKCONFIG_FILE)) test_cases = [] - with open("case_address.tmp", "r") as f: + with open("case_address.tmp", "rb") as f: for line in f: # process symbol table like: "3ffb4310 l O .dram0.data 00000018 test_desc_33$5010" line = line.split() @@ -235,7 +235,7 @@ class Parser(object): dump parsed test cases to YAML file for test bench input :param test_cases: parsed test cases """ - with open(os.path.join(self.idf_path, self.TEST_CASE_FILE), "wb+") as f: + with open(os.path.join(self.idf_path, self.TEST_CASE_FILE), "w+") as f: yaml.dump({"test cases": test_cases}, f, allow_unicode=True, default_flow_style=False) def copy_module_def_file(self): @@ -297,7 +297,7 @@ def test_parser(): } sdkconfig = ["123", "789"] tags = parser.parse_tags_internal(sdkconfig, config_dependency, parser.CONFIG_PATTERN) - assert tags == ['a', 'd', 'f'] + assert sorted(tags) == ['a', 'd', 'f'] # sorted is required for older Python3, e.g. 3.4.8 def main():