Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'feature/py3_idf_monitor' into 'master'

Browse source 
tools: Support Python3 in idf_monitor

See merge request idf/esp-idf!2930
This commit is contained in:
Angus Gratton 2018-08-27 12:29:38 +08:00
commit 3adf41a5fe
3 changed files with 62 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.gitlab-ci.yml
View file

@ -358,7 +358,12 @@ test_idf_monitor:
expire_in: 1 week expire_in: 1 week
script: script:
- cd ${IDF_PATH}/tools/test_idf_monitor - cd ${IDF_PATH}/tools/test_idf_monitor
- source /opt/pyenv/activate
- pyenv global 2.7.15
- ./run_test_idf_monitor.py - ./run_test_idf_monitor.py
- pyenv global 3.4.8
- ./run_test_idf_monitor.py
- pyenv global system
test_esp_err_to_name_on_host: test_esp_err_to_name_on_host:
<<: *host_test_template <<: *host_test_template

54
tools/idf_monitor.py
View file

@ -28,6 +28,12 @@
# Originally released under BSD-3-Clause license. # Originally released under BSD-3-Clause license.
# #
from __future__ import print_function, division from __future__ import print_function, division
from __future__ import unicode_literals
from future import standard_library
standard_library.install_aliases()
from builtins import chr
from builtins import object
from builtins import bytes
import subprocess import subprocess
import argparse import argparse
import codecs import codecs
@ -223,7 +229,7 @@ class SerialReader(StoppableThread):
except: except:
pass pass
class LineMatcher: class LineMatcher(object):
""" """
Assembles a dictionary of filtering rules based on the --print_filter Assembles a dictionary of filtering rules based on the --print_filter
argument of idf_monitor. Then later it is used to match lines and argument of idf_monitor. Then later it is used to match lines and
@ -295,7 +301,7 @@ class Monitor(object):
self.event_queue = queue.Queue() self.event_queue = queue.Queue()
self.console = miniterm.Console() self.console = miniterm.Console()
if os.name == 'nt': if os.name == 'nt':
sys.stderr = ANSIColorConverter(sys.stderr) sys.stderr = ANSIColorConverter(sys.stderr, decode_output=True)
self.console.output = ANSIColorConverter(self.console.output) self.console.output = ANSIColorConverter(self.console.output)
self.console.byte_output = ANSIColorConverter(self.console.byte_output) self.console.byte_output = ANSIColorConverter(self.console.byte_output)
@ -303,8 +309,8 @@ class Monitor(object):
# Use Console.getkey implementation from 3.3.0 (to be in sync with the ConsoleReader._cancel patch above) # Use Console.getkey implementation from 3.3.0 (to be in sync with the ConsoleReader._cancel patch above)
def getkey_patched(self): def getkey_patched(self):
c = self.enc_stdin.read(1) c = self.enc_stdin.read(1)
if c == unichr(0x7f): if c == chr(0x7f):
c = unichr(8) # map the BS key (which yields DEL) to backspace c = chr(8) # map the BS key (which yields DEL) to backspace
return c return c
self.console.getkey = types.MethodType(getkey_patched, self.console) self.console.getkey = types.MethodType(getkey_patched, self.console)
@ -320,9 +326,9 @@ class Monitor(object):
self.exit_key = CTRL_RBRACKET self.exit_key = CTRL_RBRACKET
self.translate_eol = { self.translate_eol = {
"CRLF": lambda c: c.replace(b"\n", b"\r\n"), "CRLF": lambda c: c.replace("\n", "\r\n"),
"CR": lambda c: c.replace(b"\n", b"\r"), "CR": lambda c: c.replace("\n", "\r"),
"LF": lambda c: c.replace(b"\r", b"\n"), "LF": lambda c: c.replace("\r", "\n"),
}[eol] }[eol]
# internal state # internal state
@ -404,9 +410,9 @@ class Monitor(object):
self._last_line_part = sp.pop() self._last_line_part = sp.pop()
for line in sp: for line in sp:
if line != b"": if line != b"":
if self._serial_check_exit and line == self.exit_key: if self._serial_check_exit and line == self.exit_key.encode('latin-1'):
raise SerialStopException() raise SerialStopException()
if self._output_enabled and (self._force_line_print or self._line_matcher.match(line)): if self._output_enabled and (self._force_line_print or self._line_matcher.match(line.decode(errors="ignore"))):
self.console.write_bytes(line + b'\n') self.console.write_bytes(line + b'\n')
self.handle_possible_pc_address_in_line(line) self.handle_possible_pc_address_in_line(line)
self.check_gdbstub_trigger(line) self.check_gdbstub_trigger(line)
@ -416,7 +422,7 @@ class Monitor(object):
# of the line. But after some time when we didn't received it we need # of the line. But after some time when we didn't received it we need
# to make a decision. # to make a decision.
if self._last_line_part != b"": if self._last_line_part != b"":
if self._force_line_print or (finalize_line and self._line_matcher.match(self._last_line_part)): if self._force_line_print or (finalize_line and self._line_matcher.match(self._last_line_part.decode(errors="ignore"))):
self._force_line_print = True; self._force_line_print = True;
if self._output_enabled: if self._output_enabled:
self.console.write_bytes(self._last_line_part) self.console.write_bytes(self._last_line_part)
@ -438,7 +444,7 @@ class Monitor(object):
def handle_possible_pc_address_in_line(self, line): def handle_possible_pc_address_in_line(self, line):
line = self._pc_address_buffer + line line = self._pc_address_buffer + line
self._pc_address_buffer = b"" self._pc_address_buffer = b""
for m in re.finditer(MATCH_PCADDR, line): for m in re.finditer(MATCH_PCADDR, line.decode(errors="ignore")):
self.lookup_pc_address(m.group()) self.lookup_pc_address(m.group())
def handle_menu_key(self, c): def handle_menu_key(self, c):
@ -547,8 +553,8 @@ class Monitor(object):
["%saddr2line" % self.toolchain_prefix, ["%saddr2line" % self.toolchain_prefix,
"-pfiaC", "-e", self.elf_file, pc_addr], "-pfiaC", "-e", self.elf_file, pc_addr],
cwd=".") cwd=".")
if not "?? ??:0" in translation: if not b"?? ??:0" in translation:
yellow_print(translation) yellow_print(translation.decode())
def check_gdbstub_trigger(self, line): def check_gdbstub_trigger(self, line):
line = self._gdb_buffer + line line = self._gdb_buffer + line
@ -556,7 +562,7 @@ class Monitor(object):
m = re.search(b"\\$(T..)#(..)", line) # look for a gdb "reason" for a break m = re.search(b"\\$(T..)#(..)", line) # look for a gdb "reason" for a break
if m is not None: if m is not None:
try: try:
chsum = sum(ord(p) for p in m.group(1)) & 0xFF chsum = sum(ord(bytes([p])) for p in m.group(1)) & 0xFF
calc_chsum = int(m.group(2), 16) calc_chsum = int(m.group(2), 16)
except ValueError: except ValueError:
return # payload wasn't valid hex digits return # payload wasn't valid hex digits
@ -708,13 +714,17 @@ if os.name == 'nt':
least-bad working solution, as winpty doesn't support any "passthrough" mode for raw output. least-bad working solution, as winpty doesn't support any "passthrough" mode for raw output.
""" """
def __init__(self, output): def __init__(self, output=None, decode_output=False):
self.output = output self.output = output
self.decode_output = decode_output
self.handle = GetStdHandle(STD_ERROR_HANDLE if self.output == sys.stderr else STD_OUTPUT_HANDLE) self.handle = GetStdHandle(STD_ERROR_HANDLE if self.output == sys.stderr else STD_OUTPUT_HANDLE)
self.matched = b'' self.matched = b''
def _output_write(self, data): def _output_write(self, data):
try: try:
if self.decode_output:
self.output.write(data.decode())
else:
self.output.write(data) self.output.write(data)
except IOError: except IOError:
# Windows 10 bug since the Fall Creators Update, sometimes writing to console randomly throws # Windows 10 bug since the Fall Creators Update, sometimes writing to console randomly throws
@ -723,13 +733,18 @@ if os.name == 'nt':
pass pass
def write(self, data): def write(self, data):
if type(data) is not bytes:
data = data.encode('latin-1')
for b in data: for b in data:
b = bytes([b])
l = len(self.matched) l = len(self.matched)
if b == '\033': # ESC if b == b'\033': # ESC
self.matched = b self.matched = b
elif (l == 1 and b == '[') or (1 < l < 7): elif (l == 1 and b == b'[') or (1 < l < 7):
self.matched += b self.matched += b
if self.matched == ANSI_NORMAL: # reset console if self.matched == ANSI_NORMAL.encode('latin-1'): # reset console
# Flush is required only with Python3 - switching color before it is printed would mess up the console
self.flush()
SetConsoleTextAttribute(self.handle, FOREGROUND_GREY) SetConsoleTextAttribute(self.handle, FOREGROUND_GREY)
self.matched = b'' self.matched = b''
elif len(self.matched) == 7: # could be an ANSI sequence elif len(self.matched) == 7: # could be an ANSI sequence
@ -738,6 +753,8 @@ if os.name == 'nt':
color = ANSI_TO_WINDOWS_COLOR[int(m.group(2))] color = ANSI_TO_WINDOWS_COLOR[int(m.group(2))]
if m.group(1) == b'1': if m.group(1) == b'1':
color |= FOREGROUND_INTENSITY color |= FOREGROUND_INTENSITY
# Flush is required only with Python3 - switching color before it is printed would mess up the console
self.flush()
SetConsoleTextAttribute(self.handle, color) SetConsoleTextAttribute(self.handle, color)
else: else:
self._output_write(self.matched) # not an ANSI color code, display verbatim self._output_write(self.matched) # not an ANSI color code, display verbatim
@ -749,6 +766,5 @@ if os.name == 'nt':
def flush(self): def flush(self):
self.output.flush() self.output.flush()
if __name__ == "__main__": if __name__ == "__main__":
main() main()

38
tools/test_idf_monitor/run_test_idf_monitor.py
View file

@ -14,6 +14,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
from builtins import object
from io import open
import os import os
import signal import signal
import time import time
@ -34,11 +38,11 @@ test_list = (
in_dir = 'tests/' # tests are in this directory in_dir = 'tests/' # tests are in this directory
out_dir = 'outputs/' # test results are written to this directory (kept only for debugging purposes) out_dir = 'outputs/' # test results are written to this directory (kept only for debugging purposes)
socat_in = './socatfile'# temporary socat file (deleted after run) socat_in = './socatfile'# temporary socat file (deleted after run)
monitor_error_output = out_dir + 'monitor_error_output' err_out = out_dir + 'monitor_error_output'
elf_file = './dummy.elf' # ELF file used for starting the monitor elf_file = './dummy.elf' # ELF file used for starting the monitor
idf_monitor = '{}/tools/idf_monitor.py'.format(os.getenv("IDF_PATH")) idf_monitor = '{}/tools/idf_monitor.py'.format(os.getenv("IDF_PATH"))
class SocatRunner: class SocatRunner(object):
""" """
Runs socat in the background for creating a socket. Runs socat in the background for creating a socket.
""" """
@ -54,7 +58,7 @@ class SocatRunner:
'-U', # unidirectional pipe from file to port '-U', # unidirectional pipe from file to port
'TCP4-LISTEN:2399,reuseaddr,fork', 'TCP4-LISTEN:2399,reuseaddr,fork',
'exec:"tail -c 1GB -F ' + socat_in + '"'] 'exec:"tail -c 1GB -F ' + socat_in + '"']
print ' '.join(socat_cmd) print(' '.join(socat_cmd))
self._socat_process = subprocess.Popen(socat_cmd, preexec_fn=os.setsid) # See __exit__ for os.setsid self._socat_process = subprocess.Popen(socat_cmd, preexec_fn=os.setsid) # See __exit__ for os.setsid
return self return self
@ -82,38 +86,38 @@ def main():
# another reason while the temporary socat_in file is used instead of directly reading the test files). # another reason while the temporary socat_in file is used instead of directly reading the test files).
time.sleep(1) time.sleep(1)
for t in test_list: for t in test_list:
print 'Running test on {} with filter "{}" and expecting {}'.format(t[0], t[1], t[2]) print('Running test on {} with filter "{}" and expecting {}'.format(t[0], t[1], t[2]))
with open(in_dir + t[0], "r") as input_file, open(socat_in, "w") as socat_file: with open(in_dir + t[0], "r", encoding='utf-8') as i_f, open(socat_in, "w", encoding='utf-8') as s_f:
print 'cat {} > {}'.format(input_file.name, socat_file.name) print('cat {} > {}'.format(i_f.name, s_f.name))
for line in input_file: for line in i_f:
socat_file.write(line) s_f.write(line)
idf_exit_sequence = b'\x1d\n' idf_exit_sequence = b'\x1d\n'
print 'echo "<exit>" >> {}'.format(socat_file.name) print('echo "<exit>" >> {}'.format(s_f.name))
socat_file.write(idf_exit_sequence) s_f.write(idf_exit_sequence.decode())
monitor_cmd = [idf_monitor, monitor_cmd = [idf_monitor,
'--port', 'socket://localhost:2399', '--port', 'socket://localhost:2399',
'--print_filter', t[1], '--print_filter', t[1],
elf_file] elf_file]
with open(out_dir + t[2], "w") as mon_out_f, open(monitor_error_output, "w") as mon_err_f: with open(out_dir + t[2], "w", encoding='utf-8') as o_f, open(err_out, "w", encoding='utf-8') as e_f:
try: try:
(master_fd, slave_fd) = pty.openpty() (master_fd, slave_fd) = pty.openpty()
print ' '.join(monitor_cmd), print(' '.join(monitor_cmd), end=' ')
print ' > {} 2> {} < {}'.format(mon_out_f.name, mon_err_f.name, os.ttyname(slave_fd)) print(' > {} 2> {} < {}'.format(o_f.name, e_f.name, os.ttyname(slave_fd)))
proc = subprocess.Popen(monitor_cmd, stdin=slave_fd, stdout=mon_out_f, stderr=mon_err_f, proc = subprocess.Popen(monitor_cmd, stdin=slave_fd, stdout=o_f, stderr=e_f,
close_fds=True) close_fds=True)
proc.wait() proc.wait()
finally: finally:
os.close(slave_fd) os.close(slave_fd)
os.close(master_fd) os.close(master_fd)
diff_cmd = ['diff', in_dir + t[2], out_dir + t[2]] diff_cmd = ['diff', in_dir + t[2], out_dir + t[2]]
print ' '.join(diff_cmd) print(' '.join(diff_cmd))
subprocess.check_call(diff_cmd) subprocess.check_call(diff_cmd)
print 'Test has passed' print('Test has passed')
finally: finally:
cleanup() cleanup()
end = time.time() end = time.time()
print 'Execution took {:.2f} seconds'.format(end - start) print('Execution took {:.2f} seconds'.format(end - start))
if __name__ == "__main__": if __name__ == "__main__":
main() main()