""" Test case for iperf example. This test case might have problem running on windows: 1. direct use of `make` 2. use `sudo killall iperf` to force kill iperf, didn't implement windows version The test env Example_ShieldBox do need the following config:: Example_ShieldBox: ap_list: - ssid: "ssid" password: "password" outlet: 1 apc_ip: "192.168.1.88" attenuator_port: "/dev/ttyUSB0" iperf: "/dev/ttyUSB1" apc_ip: "192.168.1.88" pc_nic: "eth0" """ from __future__ import division from __future__ import unicode_literals from builtins import str from builtins import range from builtins import object import re import os import sys import time import subprocess # add current folder to system path for importing test_report sys.path.append(os.path.dirname(__file__)) # this is a test case write with tiny-test-fw. # to run test cases outside tiny-test-fw, # we need to set environment variable `TEST_FW_PATH`, # then get and insert `TEST_FW_PATH` to sys path before import FW module test_fw_path = os.getenv("TEST_FW_PATH") if test_fw_path and test_fw_path not in sys.path: sys.path.insert(0, test_fw_path) import IDF import DUT import Utility from Utility import (Attenuator, PowerControl, LineChart) from test_report import (ThroughputForConfigsReport, ThroughputVsRssiReport) # configurations TEST_TIME = TEST_TIMEOUT = 60 WAIT_AP_POWER_ON_TIMEOUT = 90 SCAN_TIMEOUT = 3 SCAN_RETRY_COUNT = 3 RETRY_COUNT_FOR_BEST_PERFORMANCE = 2 ATTEN_VALUE_LIST = range(0, 60, 2) # constants FAILED_TO_SCAN_RSSI = -97 INVALID_HEAP_SIZE = 0xFFFFFFFF PC_IPERF_TEMP_LOG_FILE = ".tmp_iperf.log" CONFIG_NAME_PATTERN = re.compile(r"sdkconfig\.defaults\.(.+)") # We need to auto compare the difference between adjacent configs (01 -> 00, 02 -> 01, ...) and put them to reports. # Using numbers for config will make this easy. # Use default value `99` for config with best performance. BEST_PERFORMANCE_CONFIG = "99" class TestResult(object): """ record, analysis test result and convert data to output format """ PC_BANDWIDTH_LOG_PATTERN = re.compile(r"(\d+).0\s*-\s*(\d+).0\s+sec\s+[\d.]+\s+MBytes\s+([\d.]+)\s+Mbits/sec") DUT_BANDWIDTH_LOG_PATTERN = re.compile(r"(\d+)-\s+(\d+)\s+sec\s+([\d.]+)\s+Mbits/sec") ZERO_POINT_THRESHOLD = -88 # RSSI, dbm ZERO_THROUGHPUT_THRESHOLD = -92 # RSSI, dbm BAD_POINT_RSSI_THRESHOLD = -85 # RSSI, dbm BAD_POINT_MIN_THRESHOLD = 3 # Mbps BAD_POINT_PERCENTAGE_THRESHOLD = 0.3 # we need at least 1/2 valid points to qualify the test result THROUGHPUT_QUALIFY_COUNT = TEST_TIME//2 def __init__(self, proto, direction, config_name): self.proto = proto self.direction = direction self.config_name = config_name self.throughput_by_rssi = dict() self.throughput_by_att = dict() self.att_rssi_map = dict() self.heap_size = INVALID_HEAP_SIZE self.error_list = [] def _save_result(self, throughput, ap_ssid, att, rssi, heap_size): """ save the test results: * record the better throughput if att/rssi is the same. * record the min heap size. """ if ap_ssid not in self.att_rssi_map: # for new ap, create empty dict() self.throughput_by_att[ap_ssid] = dict() self.throughput_by_rssi[ap_ssid] = dict() self.att_rssi_map[ap_ssid] = dict() self.att_rssi_map[ap_ssid][att] = rssi def record_throughput(database, key_value): try: # we save the larger value for same att if throughput > database[ap_ssid][key_value]: database[ap_ssid][key_value] = throughput except KeyError: database[ap_ssid][key_value] = throughput record_throughput(self.throughput_by_att, att) record_throughput(self.throughput_by_rssi, rssi) if int(heap_size) < self.heap_size: self.heap_size = int(heap_size) def add_result(self, raw_data, ap_ssid, att, rssi, heap_size): """ add result for one test :param raw_data: iperf raw data :param ap_ssid: ap ssid that tested :param att: attenuate value :param rssi: AP RSSI :param heap_size: min heap size during test :return: throughput """ fall_to_0_recorded = 0 throughput_list = [] result_list = self.PC_BANDWIDTH_LOG_PATTERN.findall(raw_data) if not result_list: # failed to find raw data by PC pattern, it might be DUT pattern result_list = self.DUT_BANDWIDTH_LOG_PATTERN.findall(raw_data) for result in result_list: if int(result[1]) - int(result[0]) != 1: # this could be summary, ignore this continue throughput_list.append(float(result[2])) if float(result[2]) == 0 and rssi > self.ZERO_POINT_THRESHOLD \ and fall_to_0_recorded < 1: # throughput fall to 0 error. we only record 1 records for one test self.error_list.append("[Error][fall to 0][{}][att: {}][rssi: {}]: 0 throughput interval: {}-{}" .format(ap_ssid, att, rssi, result[0], result[1])) fall_to_0_recorded += 1 if len(throughput_list) > self.THROUGHPUT_QUALIFY_COUNT: throughput = sum(throughput_list) / len(throughput_list) else: throughput = 0.0 if throughput == 0 and rssi > self.ZERO_THROUGHPUT_THRESHOLD: self.error_list.append("[Error][Fatal][{}][att: {}][rssi: {}]: No throughput data found" .format(ap_ssid, att, rssi)) self._save_result(throughput, ap_ssid, att, rssi, heap_size) return throughput def post_analysis(self): """ some rules need to be checked after we collected all test raw data: 1. throughput value 30% worse than the next point with lower RSSI 2. throughput value 30% worse than the next point with larger attenuate """ def analysis_bad_point(data, index_type): for ap_ssid in data: result_dict = data[ap_ssid] index_list = list(result_dict.keys()) index_list.sort() if index_type == "att": index_list.reverse() for i, index_value in enumerate(index_list[1:]): if index_value < self.BAD_POINT_RSSI_THRESHOLD or \ result_dict[index_list[i]] < self.BAD_POINT_MIN_THRESHOLD: continue _percentage = result_dict[index_value] / result_dict[index_list[i]] if _percentage < 1 - self.BAD_POINT_PERCENTAGE_THRESHOLD: self.error_list.append("[Error][Bad point][{}][{}: {}]: drop {:.02f}%" .format(ap_ssid, index_type, index_value, (1 - _percentage) * 100)) analysis_bad_point(self.throughput_by_rssi, "rssi") analysis_bad_point(self.throughput_by_att, "att") @staticmethod def _convert_to_draw_format(data, label): keys = data.keys() keys.sort() return { "x-axis": keys, "y-axis": [data[x] for x in keys], "label": label, } def draw_throughput_figure(self, path, ap_ssid, draw_type): """ :param path: folder to save figure. make sure the folder is already created. :param ap_ssid: ap ssid string or a list of ap ssid string :param draw_type: "att" or "rssi" :return: file_name """ if draw_type == "rssi": type_name = "RSSI" data = self.throughput_by_rssi elif draw_type == "att": type_name = "Att" data = self.throughput_by_att else: raise AssertionError("draw type not supported") if isinstance(ap_ssid, list): file_name = "ThroughputVs{}_{}_{}_{}.png".format(type_name, self.proto, self.direction, hash(ap_ssid)[:6]) data_list = [self._convert_to_draw_format(data[_ap_ssid], _ap_ssid) for _ap_ssid in ap_ssid] else: file_name = "ThroughputVs{}_{}_{}_{}.png".format(type_name, self.proto, self.direction, ap_ssid) data_list = [self._convert_to_draw_format(data[ap_ssid], ap_ssid)] LineChart.draw_line_chart(os.path.join(path, file_name), "Throughput Vs {} ({} {})".format(type_name, self.proto, self.direction), "Throughput (Mbps)", "{} (dbm)".format(type_name), data_list) return file_name def draw_rssi_vs_att_figure(self, path, ap_ssid): """ :param path: folder to save figure. make sure the folder is already created. :param ap_ssid: ap to use :return: file_name """ if isinstance(ap_ssid, list): file_name = "AttVsRSSI_{}.png".format(hash(ap_ssid)[:6]) data_list = [self._convert_to_draw_format(self.att_rssi_map[_ap_ssid], _ap_ssid) for _ap_ssid in ap_ssid] else: file_name = "AttVsRSSI_{}.png".format(ap_ssid) data_list = [self._convert_to_draw_format(self.att_rssi_map[ap_ssid], ap_ssid)] LineChart.draw_line_chart(os.path.join(path, file_name), "Att Vs RSSI", "Att (dbm)", "RSSI (dbm)", data_list) return file_name def get_best_throughput(self): """ get the best throughput during test """ best_for_aps = [max(self.throughput_by_att[ap_ssid].values()) for ap_ssid in self.throughput_by_att] return max(best_for_aps) def __str__(self): """ returns summary for this test: 1. test result (success or fail) 2. best performance for each AP 3. min free heap size during test """ if self.throughput_by_att: ret = "[{}_{}][{}]: {}\r\n\r\n".format(self.proto, self.direction, self.config_name, "Fail" if self.error_list else "Success") ret += "Performance for each AP:\r\n" for ap_ssid in self.throughput_by_att: ret += "[{}]: {:.02f} Mbps\r\n".format(ap_ssid, max(self.throughput_by_att[ap_ssid].values())) if self.heap_size != INVALID_HEAP_SIZE: ret += "Minimum heap size: {}".format(self.heap_size) else: ret = "" return ret class IperfTestUtility(object): """ iperf test implementation """ def __init__(self, dut, config_name, ap_ssid, ap_password, pc_nic_ip, pc_iperf_log_file, test_result=None): self.config_name = config_name self.dut = dut self.pc_iperf_log_file = pc_iperf_log_file self.ap_ssid = ap_ssid self.ap_password = ap_password self.pc_nic_ip = pc_nic_ip if test_result: self.test_result = test_result else: self.test_result = { "tcp_tx": TestResult("tcp", "tx", config_name), "tcp_rx": TestResult("tcp", "rx", config_name), "udp_tx": TestResult("udp", "tx", config_name), "udp_rx": TestResult("udp", "rx", config_name), } def setup(self): """ setup iperf test: 1. kill current iperf process 2. reboot DUT (currently iperf is not very robust, need to reboot DUT) 3. scan to get AP RSSI 4. connect to AP """ try: subprocess.check_output("sudo killall iperf 2>&1 > /dev/null", shell=True) except subprocess.CalledProcessError: pass self.dut.write("restart") self.dut.expect("esp32>") self.dut.write("scan {}".format(self.ap_ssid)) for _ in range(SCAN_RETRY_COUNT): try: rssi = int(self.dut.expect(re.compile(r"\[{}]\[rssi=(-\d+)]".format(self.ap_ssid)), timeout=SCAN_TIMEOUT)[0]) break except DUT.ExpectTimeout: continue else: raise AssertionError("Failed to scan AP") self.dut.write("sta {} {}".format(self.ap_ssid, self.ap_password)) dut_ip = self.dut.expect(re.compile(r"event: sta ip: ([\d.]+), mask: ([\d.]+), gw: ([\d.]+)"))[0] return dut_ip, rssi def _save_test_result(self, test_case, raw_data, att, rssi, heap_size): return self.test_result[test_case].add_result(raw_data, self.ap_ssid, att, rssi, heap_size) def _test_once(self, proto, direction): """ do measure once for one type """ # connect and scan to get RSSI dut_ip, rssi = self.setup() assert direction in ["rx", "tx"] assert proto in ["tcp", "udp"] # run iperf test if direction == "tx": with open(PC_IPERF_TEMP_LOG_FILE, "w") as f: if proto == "tcp": process = subprocess.Popen(["iperf", "-s", "-B", self.pc_nic_ip, "-t", str(TEST_TIME), "-i", "1", "-f", "m"], stdout=f, stderr=f) self.dut.write("iperf -c {} -i 1 -t {}".format(self.pc_nic_ip, TEST_TIME)) else: process = subprocess.Popen(["iperf", "-s", "-u", "-B", self.pc_nic_ip, "-t", str(TEST_TIME), "-i", "1", "-f", "m"], stdout=f, stderr=f) self.dut.write("iperf -c {} -u -i 1 -t {}".format(self.pc_nic_ip, TEST_TIME)) for _ in range(TEST_TIMEOUT): if process.poll() is not None: break time.sleep(1) else: process.terminate() with open(PC_IPERF_TEMP_LOG_FILE, "r") as f: pc_raw_data = server_raw_data = f.read() else: with open(PC_IPERF_TEMP_LOG_FILE, "w") as f: if proto == "tcp": self.dut.write("iperf -s -i 1 -t {}".format(TEST_TIME)) process = subprocess.Popen(["iperf", "-c", dut_ip, "-t", str(TEST_TIME), "-f", "m"], stdout=f, stderr=f) else: self.dut.write("iperf -s -u -i 1 -t {}".format(TEST_TIME)) process = subprocess.Popen(["iperf", "-c", dut_ip, "-u", "-b", "100M", "-t", str(TEST_TIME), "-f", "m"], stdout=f, stderr=f) for _ in range(TEST_TIMEOUT): if process.poll() is not None: break time.sleep(1) else: process.terminate() server_raw_data = self.dut.read() with open(PC_IPERF_TEMP_LOG_FILE, "r") as f: pc_raw_data = f.read() # save PC iperf logs to console with open(self.pc_iperf_log_file, "a+") as f: f.write("## [{}] `{}`\r\n##### {}" .format(self.config_name, "{}_{}".format(proto, direction), time.strftime("%m-%d %H:%M:%S", time.localtime(time.time())))) f.write('\r\n```\r\n\r\n' + pc_raw_data + '\r\n```\r\n') self.dut.write("heap") heap_size = self.dut.expect(re.compile(r"min heap size: (\d+)\D"))[0] # return server raw data (for parsing test results) and RSSI return server_raw_data, rssi, heap_size def run_test(self, proto, direction, atten_val): """ run test for one type, with specified atten_value and save the test result :param proto: tcp or udp :param direction: tx or rx :param atten_val: attenuate value """ rssi = FAILED_TO_SCAN_RSSI heap_size = INVALID_HEAP_SIZE try: server_raw_data, rssi, heap_size = self._test_once(proto, direction) throughput = self._save_test_result("{}_{}".format(proto, direction), server_raw_data, atten_val, rssi, heap_size) Utility.console_log("[{}][{}_{}][{}][{}]: {:.02f}" .format(self.config_name, proto, direction, rssi, self.ap_ssid, throughput)) except Exception as e: self._save_test_result("{}_{}".format(proto, direction), "", atten_val, rssi, heap_size) Utility.console_log("Failed during test: {}".format(e)) def run_all_cases(self, atten_val): """ run test for all types (udp_tx, udp_rx, tcp_tx, tcp_rx). :param atten_val: attenuate value """ self.run_test("tcp", "tx", atten_val) self.run_test("tcp", "rx", atten_val) self.run_test("udp", "tx", atten_val) self.run_test("udp", "rx", atten_val) def wait_ap_power_on(self): """ AP need to take sometime to power on. It changes for different APs. This method will scan to check if the AP powers on. :return: True or False """ self.dut.write("restart") self.dut.expect("esp32>") for _ in range(WAIT_AP_POWER_ON_TIMEOUT // SCAN_TIMEOUT): try: self.dut.write("scan {}".format(self.ap_ssid)) self.dut.expect(re.compile(r"\[{}]\[rssi=(-\d+)]".format(self.ap_ssid)), timeout=SCAN_TIMEOUT) ret = True break except DUT.ExpectTimeout: pass else: ret = False return ret def build_iperf_with_config(config_name): """ we need to build iperf example with different configurations. :param config_name: sdkconfig we want to build """ # switch to iperf example path before build when we're running test with Runner example_path = os.path.dirname(__file__) cwd = os.getcwd() if cwd != example_path and example_path: os.chdir(example_path) try: subprocess.check_call("make clean > /dev/null", shell=True) subprocess.check_call(["cp", "sdkconfig.defaults.{}".format(config_name), "sdkconfig.defaults"]) subprocess.check_call(["rm", "-f", "sdkconfig"]) subprocess.check_call("make defconfig > /dev/null", shell=True) # save sdkconfig to generate config comparision report subprocess.check_call(["cp", "sdkconfig", "sdkconfig.{}".format(config_name)]) subprocess.check_call("make -j5 > /dev/null", shell=True) subprocess.check_call("make print_flash_cmd | tail -n 1 > build/download.config", shell=True) finally: os.chdir(cwd) def get_configs(env): att_port = env.get_variable("attenuator_port") ap_list = env.get_variable("ap_list") pc_nic_ip = env.get_pc_nic_info("pc_nic", "ipv4")["addr"] apc_ip = env.get_variable("apc_ip") pc_iperf_log_file = os.path.join(env.log_path, "pc_iperf_log.md") return att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file @IDF.idf_example_test(env_tag="Example_ShieldBox", category="stress") def test_wifi_throughput_with_different_configs(env, extra_data): """ steps: | 1. build iperf with specified configs 2. test throughput for all routers """ att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env) ap_info = ap_list[0] config_names_raw = subprocess.check_output(["ls", os.path.dirname(os.path.abspath(__file__))]) test_result = dict() sdkconfig_files = dict() for config_name in CONFIG_NAME_PATTERN.findall(config_names_raw): # 1. build config build_iperf_with_config(config_name) sdkconfig_files[config_name] = os.path.join(os.path.dirname(__file__), "sdkconfig.{}".format(config_name)) # 2. get DUT and download dut = env.get_dut("iperf", "examples/wifi/iperf") dut.start_app() dut.expect("esp32>") # 3. run test for each required att value test_result[config_name] = { "tcp_tx": TestResult("tcp", "tx", config_name), "tcp_rx": TestResult("tcp", "rx", config_name), "udp_tx": TestResult("udp", "tx", config_name), "udp_rx": TestResult("udp", "rx", config_name), } test_utility = IperfTestUtility(dut, config_name, ap_info["ssid"], ap_info["password"], pc_nic_ip, pc_iperf_log_file, test_result[config_name]) PowerControl.Control.control_rest(apc_ip, ap_info["outlet"], "OFF") PowerControl.Control.control(apc_ip, {ap_info["outlet"]: "ON"}) assert Attenuator.set_att(att_port, 0) is True if not test_utility.wait_ap_power_on(): Utility.console_log("[{}] failed to power on, skip testing this AP" .format(ap_info["ssid"]), color="red") for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE): test_utility.run_all_cases(0) for result_type in test_result[config_name]: summary = str(test_result[config_name][result_type]) if summary: Utility.console_log(summary, color="orange") # 4. check test results env.close_dut("iperf") # 5. generate report report = ThroughputForConfigsReport(os.path.join(env.log_path, "ThroughputForConfigsReport"), ap_info["ssid"], test_result, sdkconfig_files) report.generate_report() @IDF.idf_example_test(env_tag="Example_ShieldBox", category="stress") def test_wifi_throughput_vs_rssi(env, extra_data): """ steps: | 1. build with best performance config 2. switch on one router 3. set attenuator value from 0-60 for each router 4. test TCP tx rx and UDP tx rx throughput """ att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env) pc_iperf_log_file = os.path.join(env.log_path, "pc_iperf_log.md") test_result = { "tcp_tx": TestResult("tcp", "tx", BEST_PERFORMANCE_CONFIG), "tcp_rx": TestResult("tcp", "rx", BEST_PERFORMANCE_CONFIG), "udp_tx": TestResult("udp", "tx", BEST_PERFORMANCE_CONFIG), "udp_rx": TestResult("udp", "rx", BEST_PERFORMANCE_CONFIG), } # 1. build config build_iperf_with_config(BEST_PERFORMANCE_CONFIG) # 2. get DUT and download dut = env.get_dut("iperf", "examples/wifi/iperf") dut.start_app() dut.expect("esp32>") # 3. run test for each required att value for ap_info in ap_list: test_utility = IperfTestUtility(dut, BEST_PERFORMANCE_CONFIG, ap_info["ssid"], ap_info["password"], pc_nic_ip, pc_iperf_log_file, test_result) PowerControl.Control.control_rest(apc_ip, ap_info["outlet"], "OFF") PowerControl.Control.control(apc_ip, {ap_info["outlet"]: "ON"}) Attenuator.set_att(att_port, 0) if not test_utility.wait_ap_power_on(): Utility.console_log("[{}] failed to power on, skip testing this AP" .format(ap_info["ssid"]), color="red") continue for atten_val in ATTEN_VALUE_LIST: assert Attenuator.set_att(att_port, atten_val) is True test_utility.run_all_cases(atten_val) # 4. check test results env.close_dut("iperf") # 5. generate report report = ThroughputVsRssiReport(os.path.join(env.log_path, "ThroughputVsRssiReport"), test_result) report.generate_report() @IDF.idf_example_test(env_tag="Example_ShieldBox") def test_wifi_throughput_basic(env, extra_data): """ steps: | 1. test TCP tx rx and UDP tx rx throughput 2. compare with the pre-defined pass standard """ att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env) ap_info = ap_list[0] # 1. build iperf with best config build_iperf_with_config(BEST_PERFORMANCE_CONFIG) # 2. get DUT dut = env.get_dut("iperf", "examples/wifi/iperf") dut.start_app() dut.expect("esp32>") # 3. preparing test_result = { "tcp_tx": TestResult("tcp", "tx", BEST_PERFORMANCE_CONFIG), "tcp_rx": TestResult("tcp", "rx", BEST_PERFORMANCE_CONFIG), "udp_tx": TestResult("udp", "tx", BEST_PERFORMANCE_CONFIG), "udp_rx": TestResult("udp", "rx", BEST_PERFORMANCE_CONFIG), } test_utility = IperfTestUtility(dut, BEST_PERFORMANCE_CONFIG, ap_info["ssid"], ap_info["password"], pc_nic_ip, pc_iperf_log_file, test_result) PowerControl.Control.control_rest(apc_ip, ap_info["outlet"], "OFF") PowerControl.Control.control(apc_ip, {ap_info["outlet"]: "ON"}) assert Attenuator.set_att(att_port, 0) is True if not test_utility.wait_ap_power_on(): Utility.console_log("[{}] failed to power on, skip testing this AP" .format(ap_info["ssid"]), color="red") # 4. run test for TCP Tx, Rx and UDP Tx, Rx for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE): test_utility.run_all_cases(0) # 5. log performance and compare with pass standard for throughput_type in test_result: IDF.log_performance("{}_throughput".format(throughput_type), "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput())) # do check after logging, otherwise test will exit immediately if check fail, some performance can't be logged. for throughput_type in test_result: IDF.check_performance("{}_throughput".format(throughput_type), test_result[throughput_type].get_best_throughput()) env.close_dut("iperf") if __name__ == '__main__': test_wifi_throughput_basic(env_config_file="EnvConfig.yml") test_wifi_throughput_with_different_configs(env_config_file="EnvConfig.yml") test_wifi_throughput_vs_rssi(env_config_file="EnvConfig.yml")