From: Sergey Matveev Date: Sun, 22 Mar 2020 11:46:38 +0000 (+0300) Subject: ASN.1 browser X-Git-Tag: 7.4^0 X-Git-Url: http://www.git.cypherpunks.ru/?p=pyderasn.git;a=commitdiff_plain;h=a8f8eb8f0bf0395b8fc94a1cad063def57729a90 ASN.1 browser --- diff --git a/doc/features.rst b/doc/features.rst index ff913bb..5af3b37 100644 --- a/doc/features.rst +++ b/doc/features.rst @@ -71,3 +71,9 @@ Also there is `asn1crypto `__. An example of pretty printed X.509 certificate with automatically parsed DEFINED BY fields. +* :ref:`ASN.1 browser ` + + .. figure:: browser.png + :alt: ASN.1 browser example + + An example of browser running. diff --git a/doc/index.rst b/doc/index.rst index 17dcef0..b526a76 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -18,12 +18,6 @@ README), it is still often can be seen anywhere in our life. PyDERASN is `free software `__, licenced under `GNU LGPLv3 `__. -.. figure:: pprinting.png - :alt: Pretty printing example output - - An example of pretty printed X.509 certificate with automatically - parsed DEFINED BY fields. - .. toctree:: :maxdepth: 1 @@ -37,3 +31,14 @@ licenced under `GNU LGPLv3 `__. download thanks feedback + +.. figure:: pprinting.png + :alt: Pretty printing example output + + An example of pretty printed X.509 certificate with automatically + parsed DEFINED BY fields. + +.. figure:: browser.png + :alt: ASN.1 browser example + + An example of browser running. diff --git a/doc/install.rst b/doc/install.rst index d4e0abe..8aa3bd0 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -15,6 +15,8 @@ signature from `official website `__:: PyDERASN depends on `six `__ package for keeping compatibility with Py27/Py35. It is included in the tarball. You can also find it mirrored on :ref:`download ` page. +``termcolor`` is an optional dependency used for output colourizing. +``urwid`` is an optional dependency used for :ref:`interactive browser `. You could use pip (**no** OpenPGP authentication is performed!) with PyPI:: diff --git a/doc/news.rst b/doc/news.rst index ca8f5a4..997b3cb 100644 --- a/doc/news.rst +++ b/doc/news.rst @@ -8,6 +8,11 @@ News * Fix DEFINED BY pprinting when invoking as __main__ module * Integer has ``tohex()`` for getting hexadecimal representation of its value +* ``hexdump()`` (``hexdump -C`` like output) and ``ascii_visualize`` + (visualize ASCII printable characters, like in ``hexdump -C``) pretty + printing functions appeared +* Experimental ASN.1 interactive terminal browser. + You will need `urwid package `__ to use it .. _release7.3: diff --git a/pyderasn.py b/pyderasn.py index 162c140..7ce0565 100755 --- a/pyderasn.py +++ b/pyderasn.py @@ -901,6 +901,12 @@ encoding mode. If you want to encode to the memory, then you can use convenient :py:func:`pyderasn.encode2pass` helper. +.. _browser: + +ASN.1 browser +------------- +.. autofunction:: pyderasn.browse + Base Obj -------- .. autoclass:: pyderasn.Obj @@ -1011,12 +1017,14 @@ Various .. autofunction:: pyderasn.abs_decode_path .. autofunction:: pyderasn.agg_octet_string +.. autofunction:: pyderasn.ascii_visualize .. autofunction:: pyderasn.colonize_hex .. autofunction:: pyderasn.encode2pass .. autofunction:: pyderasn.encode_cer .. autofunction:: pyderasn.file_mmaped .. autofunction:: pyderasn.hexenc .. autofunction:: pyderasn.hexdec +.. autofunction:: pyderasn.hexdump .. autofunction:: pyderasn.tag_encode .. autofunction:: pyderasn.tag_decode .. autofunction:: pyderasn.tag_ctxp @@ -2300,6 +2308,15 @@ def colonize_hex(hexed): return ":".join(hexed[i:i + 2] for i in six_xrange(0, len(hexed), 2)) +def find_oid_name(asn1_type_name, oid_maps, value): + if len(oid_maps) > 0 and asn1_type_name == ObjectIdentifier.asn1_type_name: + for oid_map in oid_maps: + oid_name = oid_map.get(value) + if oid_name is not None: + return oid_name + return None + + def pp_console_row( pp, oid_maps=(), @@ -2323,9 +2340,7 @@ def pp_console_row( col += _colourize("B", "red", with_colours) if pp.bered else " " cols.append(col) col = "[%d,%d,%4d]%s" % ( - pp.tlen, - pp.llen, - pp.vlen, + pp.tlen, pp.llen, pp.vlen, LENINDEF_PP_CHAR if pp.lenindef else " " ) col = _colourize(col, "green", with_colours, ()) @@ -2337,19 +2352,11 @@ def pp_console_row( if isinstance(ent, DecodePathDefBy): cols.append(_colourize("DEFINED BY", "red", with_colours, ("reverse",))) value = str(ent.defined_by) - oid_name = None - if ( - len(oid_maps) > 0 and - ent.defined_by.asn1_type_name == - ObjectIdentifier.asn1_type_name - ): - for oid_map in oid_maps: - oid_name = oid_map.get(value) - if oid_name is not None: - cols.append(_colourize("%s:" % oid_name, "green", with_colours)) - break + oid_name = find_oid_name(ent.defined_by.asn1_type_name, oid_maps, value) if oid_name is None: cols.append(_colourize("%s:" % value, "white", with_colours, ("reverse",))) + else: + cols.append(_colourize("%s:" % oid_name, "green", with_colours)) else: cols.append(_colourize("%s:" % ent, "yellow", with_colours, ("reverse",))) if pp.expl is not None: @@ -2368,15 +2375,9 @@ def pp_console_row( if pp.value is not None: value = pp.value cols.append(_colourize(value, "white", with_colours, ("reverse",))) - if ( - len(oid_maps) > 0 and - pp.asn1_type_name == ObjectIdentifier.asn1_type_name - ): - for oid_map in oid_maps: - oid_name = oid_map.get(value) - if oid_name is not None: - cols.append(_colourize("(%s)" % oid_name, "green", with_colours)) - break + oid_name = find_oid_name(pp.asn1_type_name, oid_maps, pp.value) + if oid_name is not None: + cols.append(_colourize("(%s)" % oid_name, "green", with_colours)) if pp.asn1_type_name == Integer.asn1_type_name: cols.append(_colourize( "(%s)" % colonize_hex(pp.obj.tohex()), "green", with_colours, @@ -2425,7 +2426,7 @@ def pprint( """Pretty print object :param Obj obj: object you want to pretty print - :param oid_maps: list of ``str(OID) <-> human readable string`` dictionary. + :param oid_maps: list of ``str(OID) <-> human readable string`` dictionaries. Its human readable form is printed when OID is met :param big_blobs: if large binary objects are met (like OctetString values), do we need to print them too, on separate @@ -7404,6 +7405,392 @@ def generic_decoder(): # pragma: no cover return SEQUENCEOF(), pprint_any +def ascii_visualize(ba): + """Output only ASCII printable characters, like in hexdump -C + + Example output for given binary string (right part):: + + 92 2b 39 20 65 91 e6 8e 95 93 1a 58 df 02 78 ea |.+9 e......X..x.| + ^^^^^^^^^^^^^^^^ + """ + return "".join((chr(b) if 0x20 <= b <= 0x7E else ".") for b in ba) + + +def hexdump(raw): + """Generate ``hexdump -C`` like output + + Rendered example:: + + 00000000 30 80 30 80 a0 80 02 01 02 00 00 02 14 54 a5 18 |0.0..........T..| + 00000010 69 ef 8b 3f 15 fd ea ad bd 47 e0 94 81 6b 06 6a |i..?.....G...k.j| + + Result of that function is a generator of lines, where each line is + a list of columns:: + + [ + [...], + ["00000010 ", " 69", " ef", " 8b", " 3f", " 15", " fd", " ea", " ad ", + " bd", " 47", " e0", " 94", " 81", " 6b", " 06", " 6a ", + " |i..?.....G...k.j|"] + [...], + ] + """ + hexed = hexenc(raw).upper() + addr, cols = 0, ["%08x " % 0] + for i in six_xrange(0, len(hexed), 2): + if i != 0 and i // 2 % 8 == 0: + cols[-1] += " " + if i != 0 and i // 2 % 16 == 0: + cols.append(" |%s|" % ascii_visualize(bytearray(raw[addr:addr + 16]))) + yield cols + addr += 16 + cols = ["%08x " % addr] + cols.append(" " + hexed[i:i + 2]) + if len(cols) > 0: + cols.append(" |%s|" % ascii_visualize(bytearray(raw[addr:]))) + yield cols + + +def browse(raw, obj, oid_maps=()): + """Interactive browser + + :param bytes raw: binary data you decoded + :param obj: decoded :py:class:`pyderasn.Obj` + :param oid_maps: list of ``str(OID) <-> human readable string`` dictionaries. + Its human readable form is printed when OID is met + + .. note:: `urwid `__ dependency required + + This browser is an interactive terminal application for browsing + structures of your decoded ASN.1 objects. You can quit it with **q** + key. It consists of three windows: + + :tree: + View of ASN.1 elements hierarchy. You can navigate it using **Up**, + **Down**, **PageUp**, **PageDown**, **Home**, **End** keys. + **Left** key goes to constructed element above. **Plus**/**Minus** + keys collapse/uncollapse constructed elements. **Space** toggles it + :info: + window with various information about element. You can scroll it + with **h**/**l** (down, up) (**H**/**L** for triple speed) keys + :hexdump: + window with raw data hexdump and highlighted current element's + contents. It automatically focuses on element's data. You can + scroll it with **j**/**k** (down, up) (**J**/**K** for triple + speed) keys. If element has explicit tag, then it also will be + highlighted with different colour + + Window's header contains current decode path and progress bars with + position in *info* and *hexdump* windows. + + If you press **d**, then current element will be saved in the + current directory under its decode path name (adding ".0", ".1", etc + suffix if such file already exists). **D** will save it with explicit tag. + """ + from copy import deepcopy + from os.path import exists as path_exists + import urwid + + class TW(urwid.TreeWidget): + def __init__(self, state, *args, **kwargs): + self.state = state + self.scrolled = {"info": False, "hexdump": False} + super(TW, self).__init__(*args, **kwargs) + + def _get_pp(self): + pp = self.get_node().get_value() + constructed = len(pp) > 1 + return (pp if hasattr(pp, "_fields") else pp[0]), constructed + + def _state_update(self): + pp, _ = self._get_pp() + self.state["decode_path"].set_text( + ":".join(str(p) for p in pp.decode_path) + ) + lines = deepcopy(self.state["hexed"]) + + def attr_set(i, attr): + line = lines[i // 16] + idx = 1 + (i - 16 * (i // 16)) + line[idx] = (attr, line[idx]) + + if pp.expl_offset is not None: + for i in six_xrange( + pp.expl_offset, + pp.expl_offset + pp.expl_tlen + pp.expl_llen, + ): + attr_set(i, "select-expl") + for i in six_xrange(pp.offset, pp.offset + pp.tlen + pp.llen + pp.vlen): + attr_set(i, "select-value") + self.state["hexdump"]._set_body([urwid.Text(line) for line in lines]) + self.state["hexdump"].set_focus(pp.offset // 16) + self.state["hexdump"].set_focus_valign("middle") + self.state["hexdump_bar"].set_completion( + (100 * pp.offset // 16) // + len(self.state["hexdump"]._body.positions()) + ) + + lines = [ + [("header", "Name: "), pp.obj_name], + [("header", "Type: "), pp.asn1_type_name], + [("header", "Offset: "), "%d (0x%x)" % (pp.offset, pp.offset)], + [("header", "[TLV]len: "), "%d/%d/%d" % ( + pp.tlen, pp.llen, pp.vlen, + )], + [("header", "Slice: "), "[%d:%d]" % ( + pp.offset, pp.offset + pp.tlen + pp.llen + pp.vlen, + )], + ] + if pp.lenindef: + lines.append([("warning", "LENINDEF")]) + if pp.ber_encoded: + lines.append([("warning", "BER encoded")]) + if pp.bered: + lines.append([("warning", "BERed")]) + if pp.expl is not None: + lines.append([("header", "EXPLICIT")]) + klass, _, num = pp.expl + lines.append([" Tag: %s%d" % (TagClassReprs[klass], num)]) + if pp.expl_offset is not None: + lines.append([" Offset: %d" % pp.expl_offset]) + lines.append([" [TLV]len: %d/%d/%d" % ( + pp.expl_tlen, pp.expl_llen, pp.expl_vlen, + )]) + lines.append([" Slice: [%d:%d]" % ( + pp.expl_offset, + pp.expl_offset + pp.expl_tlen + pp.expl_llen + pp.expl_vlen, + )]) + if pp.impl is not None: + klass, _, num = pp.impl + lines.append([ + ("header", "IMPLICIT: "), "%s%d" % (TagClassReprs[klass], num), + ]) + if pp.optional: + lines.append(["OPTIONAL"]) + if pp.default: + lines.append(["DEFAULT"]) + if len(pp.decode_path) > 0: + ent = pp.decode_path[-1] + if isinstance(ent, DecodePathDefBy): + lines.append([""]) + value = str(ent.defined_by) + oid_name = find_oid_name( + ent.defined_by.asn1_type_name, oid_maps, value, + ) + lines.append([("header", "DEFINED BY: "), "%s" % ( + value if oid_name is None + else "%s (%s)" % (oid_name, value) + )]) + lines.append([""]) + if pp.value is not None: + lines.append([("header", "Value: "), pp.value]) + if ( + len(oid_maps) > 0 and + pp.asn1_type_name == ObjectIdentifier.asn1_type_name + ): + for oid_map in oid_maps: + oid_name = oid_map.get(pp.value) + if oid_name is not None: + lines.append([("header", "Human: "), oid_name]) + break + if pp.asn1_type_name == Integer.asn1_type_name: + lines.append([ + ("header", "Decimal: "), "%d" % int(pp.obj), + ]) + lines.append([ + ("header", "Hexadecimal: "), colonize_hex(pp.obj.tohex()), + ]) + if pp.blob.__class__ == binary_type: + blob = hexenc(pp.blob).upper() + for i in six_xrange(0, len(blob), 32): + lines.append([colonize_hex(blob[i:i + 32])]) + elif pp.blob.__class__ == tuple: + lines.append([", ".join(pp.blob)]) + self.state["info"]._set_body([urwid.Text(line) for line in lines]) + self.state["info_bar"].set_completion(0) + + def selectable(self): + if self.state["widget_current"] != self: + self.state["widget_current"] = self + self.scrolled["info"] = False + self.scrolled["hexdump"] = False + self._state_update() + return super(TW, self).selectable() + + def get_display_text(self): + pp, constructed = self._get_pp() + style = "constructed" if constructed else "" + if len(pp.decode_path) == 0: + return (style, pp.obj_name) + if pp.asn1_type_name == "EOC": + return ("eoc", "EOC") + ent = pp.decode_path[-1] + if isinstance(ent, DecodePathDefBy): + value = str(ent.defined_by) + oid_name = find_oid_name( + ent.defined_by.asn1_type_name, oid_maps, value, + ) + return ("defby", "DEFBY:" + ( + value if oid_name is None else oid_name + )) + return (style, ent) + + def _scroll(self, what, step): + self.state[what]._invalidate() + pos = self.state[what].focus_position + if not self.scrolled[what]: + self.scrolled[what] = True + pos -= 2 + pos = max(0, pos + step) + pos = min(pos, len(self.state[what]._body.positions()) - 1) + self.state[what].set_focus(pos) + self.state[what].set_focus_valign("top") + self.state[what + "_bar"].set_completion( + (100 * pos) // len(self.state[what]._body.positions()) + ) + + def keypress(self, size, key): + if key == "q": + raise urwid.ExitMainLoop() + + if key == " ": + self.expanded = not self.expanded + self.update_expanded_icon() + return None + + hexdump_steps = {"j": 1, "k": -1, "J": 5, "K": -5} + if key in hexdump_steps: + self._scroll("hexdump", hexdump_steps[key]) + return None + + info_steps = {"h": 1, "l": -1, "H": 5, "L": -5} + if key in info_steps: + self._scroll("info", info_steps[key]) + return None + + if key in ("d", "D"): + pp, _ = self._get_pp() + dp = ":".join(str(p) for p in pp.decode_path) + dp = dp.replace(" ", "_") + if dp == "": + dp = "root" + if key == "d" or pp.expl_offset is None: + data = self.state["raw"][pp.offset:( + pp.offset + pp.tlen + pp.llen + pp.vlen + )] + else: + data = self.state["raw"][pp.expl_offset:( + pp.expl_offset + pp.expl_tlen + pp.expl_llen + pp.expl_vlen + )] + ctr = 0 + + def duplicate_path(dp, ctr): + if ctr == 0: + return dp + return "%s.%d" % (dp, ctr) + + while True: + if not path_exists(duplicate_path(dp, ctr)): + break + ctr += 1 + dp = duplicate_path(dp, ctr) + with open(dp, "wb") as fd: + fd.write(data) + self.state["decode_path"].set_text( + ("warning", "Saved to: " + dp) + ) + return None + return super(TW, self).keypress(size, key) + + class PN(urwid.ParentNode): + def __init__(self, state, value, *args, **kwargs): + self.state = state + if not hasattr(value, "_fields"): + value = list(value) + super(PN, self).__init__(value, *args, **kwargs) + + def load_widget(self): + return TW(self.state, self) + + def load_child_keys(self): + value = self.get_value() + if hasattr(value, "_fields"): + return [] + return range(len(value[1:])) + + def load_child_node(self, key): + return PN( + self.state, + self.get_value()[key + 1], + parent=self, + key=key, + depth=self.get_depth() + 1, + ) + + class LabeledPG(urwid.ProgressBar): + def __init__(self, label, *args, **kwargs): + self.label = label + super(LabeledPG, self).__init__(*args, **kwargs) + + def get_text(self): + return "%s: %s" % (self.label, super(LabeledPG, self).get_text()) + + WinHexdump = urwid.ListBox([urwid.Text("")]) + WinInfo = urwid.ListBox([urwid.Text("")]) + WinDecodePath = urwid.Text("", "center") + WinInfoBar = LabeledPG("info", "pg-normal", "pg-complete") + WinHexdumpBar = LabeledPG("hexdump", "pg-normal", "pg-complete") + WinTree = urwid.TreeListBox(urwid.TreeWalker(PN( + { + "raw": raw, + "hexed": list(hexdump(raw)), + "widget_current": None, + "info": WinInfo, + "info_bar": WinInfoBar, + "hexdump": WinHexdump, + "hexdump_bar": WinHexdumpBar, + "decode_path": WinDecodePath, + }, + list(obj.pps()), + ))) + help_text = " ".join(( + "q:quit", + "space:(un)collapse", + "(pg)up/down/home/end:nav", + "jkJK:hexdump hlHL:info", + "dD:dump", + )) + urwid.MainLoop( + urwid.Frame( + urwid.Columns([ + ("weight", 1, WinTree), + ("weight", 2, urwid.Pile([ + urwid.LineBox(WinInfo), + urwid.LineBox(WinHexdump), + ])), + ]), + header=urwid.Columns([ + ("weight", 2, urwid.AttrWrap(WinDecodePath, "header")), + ("weight", 1, WinInfoBar), + ("weight", 1, WinHexdumpBar), + ]), + footer=urwid.AttrWrap(urwid.Text(help_text), "help") + ), + [ + ("header", "bold", ""), + ("constructed", "bold", ""), + ("help", "light magenta", ""), + ("warning", "light red", ""), + ("defby", "light red", ""), + ("eoc", "dark red", ""), + ("select-value", "light green", ""), + ("select-expl", "light red", ""), + ("pg-normal", "", "light blue"), + ("pg-complete", "black", "yellow"), + ], + ).run() + + def main(): # pragma: no cover import argparse parser = argparse.ArgumentParser(description="PyDERASN ASN.1 BER/CER/DER decoder") @@ -7449,6 +7836,11 @@ def main(): # pragma: no cover action="store_true", help="Turn on event generation mode", ) + parser.add_argument( + "--browse", + action="store_true", + help="Start ASN.1 browser", + ) parser.add_argument( "RAWFile", type=argparse.FileType("rb"), @@ -7477,6 +7869,11 @@ def main(): # pragma: no cover } if args.defines_by_path is not None: ctx["defines_by_path"] = obj_by_path(args.defines_by_path) + if args.browse: + obj, _ = schema().decode(raw, ctx=ctx) + browse(raw, obj, oid_maps) + from sys import exit as sys_exit + sys_exit(0) from os import environ pprinter = partial( pprinter,