X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=pyderasn.py;h=f563afef86964240e57971d85c91fa35d40de0c8;hb=5699682993d2c2b789640c20cd68e7bd1eb0cf7c;hp=b15af8d0bbb502b80c2ff1e1d5deaf521279623a;hpb=94f91e21c9c17b190596cd1178afbcc0bee84758;p=pyderasn.git diff --git a/pyderasn.py b/pyderasn.py index b15af8d..f563afe 100755 --- a/pyderasn.py +++ b/pyderasn.py @@ -1,12 +1,11 @@ #!/usr/bin/env python # coding: utf-8 # PyDERASN -- Python ASN.1 DER/BER codec with abstract structures -# Copyright (C) 2017-2018 Sergey Matveev +# Copyright (C) 2017-2020 Sergey Matveev # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. +# published by the Free Software Foundation, version 3 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -14,8 +13,7 @@ # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public -# License along with this program. If not, see -# . +# License along with this program. If not, see . """Python ASN.1 DER/BER codec with abstract structures This library allows you to marshal various structures in ASN.1 DER @@ -213,9 +211,11 @@ decoding process. Currently available context options: +* :ref:`allow_default_values ` +* :ref:`allow_expl_oob ` +* :ref:`allow_unordered_set ` * :ref:`bered ` * :ref:`defines_by_path ` -* :ref:`strict_default_existence ` .. _pprinting: @@ -237,6 +237,105 @@ all object ``repr``. But it is easy to write custom formatters. >>> print(pprint(obj)) 0 [1,1, 2] INTEGER -12345 +.. _pprint_example: + +Example certificate:: + + >>> print(pprint(crt)) + 0 [1,3,1604] Certificate SEQUENCE + 4 [1,3,1453] . tbsCertificate: TBSCertificate SEQUENCE + 10-2 [1,1, 1] . . version: [0] EXPLICIT Version INTEGER v3 OPTIONAL + 13 [1,1, 3] . . serialNumber: CertificateSerialNumber INTEGER 61595 + 18 [1,1, 13] . . signature: AlgorithmIdentifier SEQUENCE + 20 [1,1, 9] . . . algorithm: OBJECT IDENTIFIER 1.2.840.113549.1.1.5 + 31 [0,0, 2] . . . parameters: [UNIV 5] ANY OPTIONAL + . . . . 05:00 + 33 [0,0, 278] . . issuer: Name CHOICE rdnSequence + 33 [1,3, 274] . . . rdnSequence: RDNSequence SEQUENCE OF + 37 [1,1, 11] . . . . 0: RelativeDistinguishedName SET OF + 39 [1,1, 9] . . . . . 0: AttributeTypeAndValue SEQUENCE + 41 [1,1, 3] . . . . . . type: AttributeType OBJECT IDENTIFIER 2.5.4.6 + 46 [0,0, 4] . . . . . . value: [UNIV 19] AttributeValue ANY + . . . . . . . 13:02:45:53 + [...] + 1461 [1,1, 13] . signatureAlgorithm: AlgorithmIdentifier SEQUENCE + 1463 [1,1, 9] . . algorithm: OBJECT IDENTIFIER 1.2.840.113549.1.1.5 + 1474 [0,0, 2] . . parameters: [UNIV 5] ANY OPTIONAL + . . . 05:00 + 1476 [1,2, 129] . signatureValue: BIT STRING 1024 bits + . . 68:EE:79:97:97:DD:3B:EF:16:6A:06:F2:14:9A:6E:CD + . . 9E:12:F7:AA:83:10:BD:D1:7C:98:FA:C7:AE:D4:0E:2C + [...] + + Trailing data: 0a + +Let's parse that output, human:: + + 10-2 [1,1, 1] . . version: [0] EXPLICIT Version INTEGER v3 OPTIONAL + ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ + 0 1 2 3 4 5 6 7 8 9 10 11 + +:: + + 20 [1,1, 9] . . . algorithm: OBJECT IDENTIFIER 1.2.840.113549.1.1.5 + ^ ^ ^ ^ ^ ^ ^ ^ + 0 2 3 4 5 6 9 10 + +:: + + 33 [0,0, 278] . . issuer: Name CHOICE rdnSequence + ^ ^ ^ ^ ^ ^ ^ ^ ^ + 0 2 3 4 5 6 8 9 10 + +:: + + 52-2∞ B [1,1,1054]∞ . . . . eContent: [0] EXPLICIT BER OCTET STRING 1046 bytes + ^ ^ ^ ^ ^ + 12 13 14 9 10 + +:0: + Offset of the object, where its DER/BER encoding begins. + Pay attention that it does **not** include explicit tag. +:1: + If explicit tag exists, then this is its length (tag + encoded length). +:2: + Length of object's tag. For example CHOICE does not have its own tag, + so it is zero. +:3: + Length of encoded length. +:4: + Length of encoded value. +:5: + Visual indentation to show the depth of object in the hierarchy. +:6: + Object's name inside SEQUENCE/CHOICE. +:7: + If either IMPLICIT or EXPLICIT tag is set, then it will be shown + here. "IMPLICIT" is omitted. +:8: + Object's class name, if set. Omitted if it is just an ordinary simple + value (like with ``algorithm`` in example above). +:9: + Object's ASN.1 type. +:10: + Object's value, if set. Can consist of multiple words (like OCTET/BIT + STRINGs above). We see ``v3`` value in Version, because it is named. + ``rdnSequence`` is the choice of CHOICE type. +:11: + Possible other flags like OPTIONAL and DEFAULT, if value equals to the + default one, specified in the schema. +:12: + Shows does object contains any kind of BER encoded data (possibly + Sequence holding BER-encoded underlying value). +:13: + Only applicable to BER encoded data. Indefinite length encoding mark. +:14: + Only applicable to BER encoded data. If object has BER-specific + encoding, then ``BER`` will be shown. It does not depend on indefinite + length encoding. ``EOC``, ``BOOLEAN``, ``BIT STRING``, ``OCTET STRING`` + (and its derivatives), ``SET``, ``SET OF`` could be BERed. + + .. _definedby: DEFINED BY @@ -247,6 +346,8 @@ DEFINED BY some previously met ObjectIdentifier. This library provides ability to specify mapping between some OID and field that must be decoded with specific specification. +.. _defines: + defines kwarg _____________ @@ -274,7 +375,7 @@ You can specify multiple fields, that will be autodecoded -- that is why ``defines`` kwarg is a sequence. You can specify defined field relatively or absolutely to current decode path. For example ``defines`` for AlgorithmIdentifier of X.509's -``tbsCertificate.subjectPublicKeyInfo.algorithm.algorithm``:: +``tbsCertificate:subjectPublicKeyInfo:algorithm:algorithm``:: ( (("parameters",), { @@ -302,8 +403,7 @@ Following types can be automatically decoded (DEFINED BY): When any of those fields is automatically decoded, then ``.defined`` attribute contains ``(OID, value)`` tuple. ``OID`` tells by which OID it was defined, ``value`` contains corresponding decoded value. For example -above, ``content_info["content"].defined == (id_signedData, -signed_data)``. +above, ``content_info["content"].defined == (id_signedData, signed_data)``. .. _defines_by_path_ctx: @@ -321,15 +421,15 @@ value must be sequence of following tuples:: where ``decode_path`` is a tuple holding so-called decode path to the exact :py:class:`pyderasn.ObjectIdentifier` field you want to apply -``defines``, holding exactly the same value as accepted in its keyword -argument. +``defines``, holding exactly the same value as accepted in its +:ref:`keyword argument `. For example, again for CMS, you want to automatically decode ``SignedData`` and CMC's (:rfc:`5272`) ``PKIData`` and ``PKIResponse`` structures it may hold. Also, automatically decode ``controlSequence`` of ``PKIResponse``:: - content_info, tail = ContentInfo().decode(data, defines_by_path=( + content_info, tail = ContentInfo().decode(data, ctx={"defines_by_path": ( ( ("contentType",), ((("content",), {id_signedData: SignedData()}),), @@ -364,7 +464,7 @@ of ``PKIResponse``:: id_cmc_transactionId: TransactionId(), })), ), - )) + )}) Pay attention for :py:class:`pyderasn.DecodePathDefBy` and ``any``. First function is useful for path construction when some automatic @@ -381,19 +481,47 @@ DER. But you can optionally enable BER decoding with setting ``bered`` :ref:`context ` argument to True. Indefinite lengths and constructed primitive types should be parsed successfully. -* If object is encoded in BER form (not the DER one), then ``bered`` +* If object is encoded in BER form (not the DER one), then ``ber_encoded`` attribute is set to True. Only ``BOOLEAN``, ``BIT STRING``, ``OCTET - STRING`` can contain it. + STRING``, ``OBJECT IDENTIFIER``, ``SEQUENCE``, ``SET``, ``SET OF`` + can contain it. * If object has an indefinite length encoding, then its ``lenindef`` attribute is set to True. Only ``BIT STRING``, ``OCTET STRING``, ``SEQUENCE``, ``SET``, ``SEQUENCE OF``, ``SET OF``, ``ANY`` can contain it. * If object has an indefinite length encoded explicit tag, then ``expl_lenindef`` is set to True. +* If object has either any of BER-related encoding (explicit tag + indefinite length, object's indefinite length, BER-encoding) or any + underlying component has that kind of encoding, then ``bered`` + attribute is set to True. For example SignedData CMS can have + ``ContentInfo:content:signerInfos:*`` ``bered`` value set to True, but + ``ContentInfo:content:signerInfos:*:signedAttrs`` won't. EOC (end-of-contents) token's length is taken in advance in object's value length. +.. _allow_expl_oob_ctx: + +Allow explicit tag out-of-bound +------------------------------- + +Invalid BER encoding could contain ``EXPLICIT`` tag containing more than +one value, more than one object. If you set ``allow_expl_oob`` context +option to True, then no error will be raised and that invalid encoding +will be silently further processed. But pay attention that offsets and +lengths will be invalid in that case. + +.. warning:: + + This option should be used only for skipping some decode errors, just + to see the decoded structure somehow. + +Base Obj +-------- +.. autoclass:: pyderasn.Obj + :members: + Primitive types --------------- @@ -439,6 +567,10 @@ NumericString _____________ .. autoclass:: pyderasn.NumericString +PrintableString +_______________ +.. autoclass:: pyderasn.PrintableString + UTCTime _______ .. autoclass:: pyderasn.UTCTime @@ -492,13 +624,13 @@ Various ------- .. autofunction:: pyderasn.abs_decode_path +.. autofunction:: pyderasn.colonize_hex .. autofunction:: pyderasn.hexenc .. autofunction:: pyderasn.hexdec .. autofunction:: pyderasn.tag_encode .. autofunction:: pyderasn.tag_decode .. autofunction:: pyderasn.tag_ctxp .. autofunction:: pyderasn.tag_ctxc -.. autoclass:: pyderasn.Obj .. autoclass:: pyderasn.DecodeError :members: __init__ .. autoclass:: pyderasn.NotEnoughData @@ -516,9 +648,11 @@ from codecs import getdecoder from codecs import getencoder from collections import namedtuple from collections import OrderedDict +from copy import copy from datetime import datetime from math import ceil from os import environ +from string import ascii_letters from string import digits from six import add_metaclass @@ -528,18 +662,22 @@ from six import indexbytes from six import int2byte from six import integer_types from six import iterbytes +from six import iteritems +from six import itervalues from six import PY2 from six import string_types from six import text_type +from six import unichr as six_unichr from six.moves import xrange as six_xrange try: from termcolor import colored -except ImportError: - def colored(what, *args): +except ImportError: # pragma: no cover + def colored(what, *args, **kwargs): return what +__version__ = "5.5" __all__ = ( "Any", @@ -618,7 +756,11 @@ LENINDEF_PP_CHAR = "I" if PY2 else "∞" # Errors ######################################################################## -class DecodeError(Exception): +class ASN1Error(ValueError): + pass + + +class DecodeError(ASN1Error): def __init__(self, msg="", klass=None, decode_path=(), offset=0): """ :param str msg: reason of decode failing @@ -641,7 +783,7 @@ class DecodeError(Exception): c for c in ( "" if self.klass is None else self.klass.__name__, ( - ("(%s)" % ".".join(str(dp) for dp in self.decode_path)) + ("(%s)" % ":".join(str(dp) for dp in self.decode_path)) if len(self.decode_path) > 0 else "" ), ("(at %d)" % self.offset) if self.offset > 0 else "", @@ -673,7 +815,7 @@ class InvalidOID(DecodeError): pass -class ObjUnknown(ValueError): +class ObjUnknown(ASN1Error): def __init__(self, name): super(ObjUnknown, self).__init__() self.name = name @@ -685,7 +827,7 @@ class ObjUnknown(ValueError): return "%s(%s)" % (self.__class__.__name__, self) -class ObjNotReady(ValueError): +class ObjNotReady(ASN1Error): def __init__(self, name): super(ObjNotReady, self).__init__() self.name = name @@ -697,7 +839,7 @@ class ObjNotReady(ValueError): return "%s(%s)" % (self.__class__.__name__, self) -class InvalidValueType(ValueError): +class InvalidValueType(ASN1Error): def __init__(self, expected_types): super(InvalidValueType, self).__init__() self.expected_types = expected_types @@ -711,7 +853,7 @@ class InvalidValueType(ValueError): return "%s(%s)" % (self.__class__.__name__, self) -class BoundsError(ValueError): +class BoundsError(ASN1Error): def __init__(self, bound_min, value, bound_max): super(BoundsError, self).__init__() self.bound_min = bound_min @@ -882,9 +1024,9 @@ def len_decode(data): ######################################################################## class AutoAddSlots(type): - def __new__(mcs, name, bases, _dict): + def __new__(cls, name, bases, _dict): _dict["__slots__"] = _dict.get("__slots__", ()) - return type.__new__(mcs, name, bases, _dict) + return type.__new__(cls, name, bases, _dict) @add_metaclass(AutoAddSlots) @@ -905,7 +1047,7 @@ class Obj(object): "vlen", "expl_lenindef", "lenindef", - "bered", + "ber_encoded", ) def __init__( @@ -927,7 +1069,7 @@ class Obj(object): self.default = None self.expl_lenindef = False self.lenindef = False - self.bered = False + self.ber_encoded = False @property def ready(self): # pragma: no cover @@ -939,6 +1081,12 @@ class Obj(object): if not self.ready: raise ObjNotReady(self.__class__.__name__) + @property + def bered(self): + """Is either object or any elements inside is BER encoded? + """ + return self.expl_lenindef or self.lenindef or self.ber_encoded + @property def decoded(self): """Is object decoded? @@ -952,10 +1100,14 @@ class Obj(object): @property def tlen(self): + """See :ref:`decoding` + """ return len(self.tag) @property def tlvlen(self): + """See :ref:`decoding` + """ return self.tlen + self.llen + self.vlen def __str__(self): # pragma: no cover @@ -980,6 +1132,10 @@ class Obj(object): raise NotImplementedError() def encode(self): + """Encode the structure + + :returns: DER representation + """ raw = self._encode() if self._expl is None: return raw @@ -993,6 +1149,7 @@ class Obj(object): decode_path=(), ctx=None, tag_only=False, + _ctx_immutable=True, ): """Decode the data @@ -1000,14 +1157,19 @@ class Obj(object): :param int offset: initial data's offset :param bool leavemm: do we need to leave memoryview of remaining data as is, or convert it to bytes otherwise - :param ctx: optional :ref:`context ` governing decoding process. + :param ctx: optional :ref:`context ` governing decoding process :param tag_only: decode only the tag, without length and contents (used only in Choice and Set structures, trying to determine if tag satisfies the scheme) + :param _ctx_immutable: do we need to copy ``ctx`` before using it :returns: (Obj, remaining data) + + .. seealso:: :ref:`decoding` """ if ctx is None: ctx = {} + elif _ctx_immutable: + ctx = copy(ctx) tlv = memoryview(data) if self._expl is None: result = self._decode( @@ -1018,7 +1180,7 @@ class Obj(object): tag_only=tag_only, ) if tag_only: - return + return None obj, tail = result else: try: @@ -1055,8 +1217,8 @@ class Obj(object): ctx=ctx, tag_only=tag_only, ) - if tag_only: - return + if tag_only: # pragma: no cover + return None obj, tail = result eoc_expected, tail = tail[:EOC_LEN], tail[EOC_LEN:] if eoc_expected.tobytes() != EOC: @@ -1090,51 +1252,79 @@ class Obj(object): ctx=ctx, tag_only=tag_only, ) - if tag_only: - return + if tag_only: # pragma: no cover + return None obj, tail = result + if obj.tlvlen < l and not ctx.get("allow_expl_oob", False): + raise DecodeError( + "explicit tag out-of-bound, longer than data", + klass=self.__class__, + decode_path=decode_path, + offset=offset, + ) return obj, (tail if leavemm else tail.tobytes()) @property def expled(self): + """See :ref:`decoding` + """ return self._expl is not None @property def expl_tag(self): + """See :ref:`decoding` + """ return self._expl @property def expl_tlen(self): + """See :ref:`decoding` + """ return len(self._expl) @property def expl_llen(self): + """See :ref:`decoding` + """ if self.expl_lenindef: return 1 return len(len_encode(self.tlvlen)) @property def expl_offset(self): + """See :ref:`decoding` + """ return self.offset - self.expl_tlen - self.expl_llen @property def expl_vlen(self): + """See :ref:`decoding` + """ return self.tlvlen @property def expl_tlvlen(self): + """See :ref:`decoding` + """ return self.expl_tlen + self.expl_llen + self.expl_vlen @property def fulloffset(self): + """See :ref:`decoding` + """ return self.expl_offset if self.expled else self.offset @property def fulllen(self): + """See :ref:`decoding` + """ return self.expl_tlvlen if self.expled else self.tlvlen def pps_lenindef(self, decode_path): - if self.lenindef: + if self.lenindef and not ( + getattr(self, "defined", None) is not None and + self.defined[1].lenindef + ): yield _pp( asn1_type_name="EOC", obj_name="", @@ -1146,6 +1336,7 @@ class Obj(object): tlen=1, llen=1, vlen=0, + ber_encoded=True, bered=True, ) if self.expl_lenindef: @@ -1157,6 +1348,7 @@ class Obj(object): tlen=1, llen=1, vlen=0, + ber_encoded=True, bered=True, ) @@ -1189,6 +1381,7 @@ class DecodePathDefBy(object): ######################################################################## PP = namedtuple("PP", ( + "obj", "asn1_type_name", "obj_name", "decode_path", @@ -1208,11 +1401,13 @@ PP = namedtuple("PP", ( "expl_vlen", "expl_lenindef", "lenindef", + "ber_encoded", "bered", )) def _pp( + obj=None, asn1_type_name="unknown", obj_name="unknown", decode_path=(), @@ -1232,9 +1427,11 @@ def _pp( expl_vlen=None, expl_lenindef=False, lenindef=False, + ber_encoded=False, bered=False, ): return PP( + obj, asn1_type_name, obj_name, decode_path, @@ -1254,20 +1451,29 @@ def _pp( expl_vlen, expl_lenindef, lenindef, + ber_encoded, bered, ) -def _colorize(what, colour, with_colours, attrs=("bold",)): +def _colourize(what, colour, with_colours, attrs=("bold",)): return colored(what, colour, attrs=attrs) if with_colours else what +def colonize_hex(hexed): + """Separate hexadecimal string with colons + """ + return ":".join(hexed[i:i + 2] for i in six_xrange(0, len(hexed), 2)) + + def pp_console_row( pp, - oids=None, + oid_maps=(), with_offsets=False, with_blob=True, with_colours=False, + with_decode_path=False, + decode_path_len_decrease=0, ): cols = [] if with_offsets: @@ -1279,113 +1485,161 @@ def pp_console_row( ), LENINDEF_PP_CHAR if pp.expl_lenindef else " ", ) - cols.append(_colorize(col, "red", with_colours, ())) + col = _colourize(col, "red", with_colours, ()) + col += _colourize("B", "red", with_colours) if pp.bered else " " + cols.append(col) col = "[%d,%d,%4d]%s" % ( pp.tlen, pp.llen, pp.vlen, LENINDEF_PP_CHAR if pp.lenindef else " " ) - col = _colorize(col, "green", with_colours, ()) + col = _colourize(col, "green", with_colours, ()) cols.append(col) - if len(pp.decode_path) > 0: - cols.append(" ." * (len(pp.decode_path))) + decode_path_len = len(pp.decode_path) - decode_path_len_decrease + if decode_path_len > 0: + cols.append(" ." * decode_path_len) ent = pp.decode_path[-1] if isinstance(ent, DecodePathDefBy): - cols.append(_colorize("DEFINED BY", "red", with_colours, ("reverse",))) + cols.append(_colourize("DEFINED BY", "red", with_colours, ("reverse",))) value = str(ent.defined_by) + oid_name = None if ( - oids is not None and + len(oid_maps) > 0 and ent.defined_by.asn1_type_name == - ObjectIdentifier.asn1_type_name and - value in oids + ObjectIdentifier.asn1_type_name ): - cols.append(_colorize("%s:" % oids[value], "green", with_colours)) - else: - cols.append(_colorize("%s:" % value, "white", with_colours, ("reverse",))) + for oid_map in oid_maps: + oid_name = oid_map.get(value) + if oid_name is not None: + cols.append(_colourize("%s:" % oid_name, "green", with_colours)) + break + if oid_name is None: + cols.append(_colourize("%s:" % value, "white", with_colours, ("reverse",))) else: - cols.append(_colorize("%s:" % ent, "yellow", with_colours, ("reverse",))) + cols.append(_colourize("%s:" % ent, "yellow", with_colours, ("reverse",))) if pp.expl is not None: klass, _, num = pp.expl col = "[%s%d] EXPLICIT" % (TagClassReprs[klass], num) - cols.append(_colorize(col, "blue", with_colours)) + cols.append(_colourize(col, "blue", with_colours)) if pp.impl is not None: klass, _, num = pp.impl col = "[%s%d]" % (TagClassReprs[klass], num) - cols.append(_colorize(col, "blue", with_colours)) + cols.append(_colourize(col, "blue", with_colours)) if pp.asn1_type_name.replace(" ", "") != pp.obj_name.upper(): - cols.append(_colorize(pp.obj_name, "magenta", with_colours)) - if pp.bered: - cols.append(_colorize("BER", "red", with_colours)) - cols.append(_colorize(pp.asn1_type_name, "cyan", with_colours)) + cols.append(_colourize(pp.obj_name, "magenta", with_colours)) + if pp.ber_encoded: + cols.append(_colourize("BER", "red", with_colours)) + cols.append(_colourize(pp.asn1_type_name, "cyan", with_colours)) if pp.value is not None: value = pp.value - cols.append(_colorize(value, "white", with_colours, ("reverse",))) + cols.append(_colourize(value, "white", with_colours, ("reverse",))) if ( - oids is not None and - pp.asn1_type_name == ObjectIdentifier.asn1_type_name and - value in oids + len(oid_maps) > 0 and + pp.asn1_type_name == ObjectIdentifier.asn1_type_name ): - cols.append(_colorize("(%s)" % oids[value], "green", with_colours)) + for oid_map in oid_maps: + oid_name = oid_map.get(value) + if oid_name is not None: + cols.append(_colourize("(%s)" % oid_name, "green", with_colours)) + break + if pp.asn1_type_name == Integer.asn1_type_name: + hex_repr = hex(int(pp.obj._value))[2:].upper() + if len(hex_repr) % 2 != 0: + hex_repr = "0" + hex_repr + cols.append(_colourize( + "(%s)" % colonize_hex(hex_repr), + "green", + with_colours, + )) if with_blob: if isinstance(pp.blob, binary_type): cols.append(hexenc(pp.blob)) elif isinstance(pp.blob, tuple): cols.append(", ".join(pp.blob)) if pp.optional: - cols.append(_colorize("OPTIONAL", "red", with_colours)) + cols.append(_colourize("OPTIONAL", "red", with_colours)) if pp.default: - cols.append(_colorize("DEFAULT", "red", with_colours)) + cols.append(_colourize("DEFAULT", "red", with_colours)) + if with_decode_path: + cols.append(_colourize( + "[%s]" % ":".join(str(p) for p in pp.decode_path), + "grey", + with_colours, + )) return " ".join(cols) -def pp_console_blob(pp): - cols = [" " * len("XXXXXYYZ [X,X,XXXX]Z")] - if len(pp.decode_path) > 0: - cols.append(" ." * (len(pp.decode_path) + 1)) +def pp_console_blob(pp, decode_path_len_decrease=0): + cols = [" " * len("XXXXXYYZZ [X,X,XXXX]Z")] + decode_path_len = len(pp.decode_path) - decode_path_len_decrease + if decode_path_len > 0: + cols.append(" ." * (decode_path_len + 1)) if isinstance(pp.blob, binary_type): blob = hexenc(pp.blob).upper() - for i in range(0, len(blob), 32): + for i in six_xrange(0, len(blob), 32): chunk = blob[i:i + 32] - yield " ".join(cols + [":".join( - chunk[j:j + 2] for j in range(0, len(chunk), 2) - )]) + yield " ".join(cols + [colonize_hex(chunk)]) elif isinstance(pp.blob, tuple): yield " ".join(cols + [", ".join(pp.blob)]) -def pprint(obj, oids=None, big_blobs=False, with_colours=False): +def pprint( + obj, + oid_maps=(), + big_blobs=False, + with_colours=False, + with_decode_path=False, + decode_path_only=(), +): """Pretty print object :param Obj obj: object you want to pretty print - :param oids: ``OID <-> humand readable string`` dictionary. When OID - from it is met, then its humand readable form is printed + :param oid_maps: list of ``OID <-> humand readable string`` dictionary. + When OID from it is met, then its humand readable form + is printed :param big_blobs: if large binary objects are met (like OctetString values), do we need to print them too, on separate lines :param with_colours: colourize output, if ``termcolor`` library is available + :param with_decode_path: print decode path + :param decode_path_only: print only that specified decode path """ def _pprint_pps(pps): for pp in pps: if hasattr(pp, "_fields"): + if ( + decode_path_only != () and + tuple( + str(p) for p in pp.decode_path[:len(decode_path_only)] + ) != decode_path_only + ): + continue if big_blobs: yield pp_console_row( pp, - oids=oids, + oid_maps=oid_maps, with_offsets=True, with_blob=False, with_colours=with_colours, + with_decode_path=with_decode_path, + decode_path_len_decrease=len(decode_path_only), ) - for row in pp_console_blob(pp): + for row in pp_console_blob( + pp, + decode_path_len_decrease=len(decode_path_only), + ): yield row else: yield pp_console_row( pp, - oids=oids, + oid_maps=oid_maps, with_offsets=True, with_blob=True, with_colours=with_colours, + with_decode_path=with_decode_path, + decode_path_len_decrease=len(decode_path_only), ) else: for row in _pprint_pps(pp): @@ -1441,10 +1695,10 @@ class Boolean(Obj): self._value = default def _value_sanitize(self, value): - if issubclass(value.__class__, Boolean): - return value._value if isinstance(value, bool): return value + if issubclass(value.__class__, Boolean): + return value._value raise InvalidValueType((self.__class__, bool)) @property @@ -1461,6 +1715,9 @@ class Boolean(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __nonzero__(self): @@ -1523,7 +1780,7 @@ class Boolean(Obj): offset=offset, ) if tag_only: - return + return None try: l, _, v = len_decode(lv) except DecodeError as err: @@ -1548,14 +1805,14 @@ class Boolean(Obj): offset=offset, ) first_octet = byte2int(v) - bered = False + ber_encoded = False if first_octet == 0: value = False elif first_octet == 0xFF: value = True elif ctx.get("bered", False): value = True - bered = True + ber_encoded = True else: raise DecodeError( "unacceptable Boolean value", @@ -1571,7 +1828,7 @@ class Boolean(Obj): optional=self.optional, _decoded=(offset, 1, 1), ) - obj.bered = bered + obj.ber_encoded = ber_encoded return obj, v[1:] def __repr__(self): @@ -1579,6 +1836,7 @@ class Boolean(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -1596,6 +1854,7 @@ class Boolean(Obj): expl_llen=self.expl_llen if self.expled else None, expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, + ber_encoded=self.ber_encoded, bered=self.bered, ) for pp in self.pps_lenindef(decode_path): @@ -1685,10 +1944,10 @@ class Integer(Obj): self._value = default def _value_sanitize(self, value): - if issubclass(value.__class__, Integer): - value = value._value - elif isinstance(value, integer_types): + if isinstance(value, integer_types): pass + elif issubclass(value.__class__, Integer): + value = value._value elif isinstance(value, str): value = self.specs.get(value) if value is None: @@ -1715,6 +1974,9 @@ class Integer(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __int__(self): @@ -1745,9 +2007,10 @@ class Integer(Obj): @property def named(self): - for name, value in self.specs.items(): + for name, value in iteritems(self.specs): if value == self._value: return name + return None def __call__( self, @@ -1827,7 +2090,7 @@ class Integer(Obj): offset=offset, ) if tag_only: - return + return None try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -1905,6 +2168,7 @@ class Integer(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -1922,11 +2186,15 @@ class Integer(Obj): expl_llen=self.expl_llen if self.expled else None, expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, + bered=self.bered, ) for pp in self.pps_lenindef(decode_path): yield pp +SET01 = frozenset(("0", "1")) + + class BitString(Obj): """``BIT STRING`` bit string type @@ -2032,8 +2300,6 @@ class BitString(Obj): return bit_len, bytes(octets) def _value_sanitize(self, value): - if issubclass(value.__class__, BitString): - return value._value if isinstance(value, (string_types, binary_type)): if ( isinstance(value, string_types) and @@ -2041,10 +2307,10 @@ class BitString(Obj): ): if value.endswith("'B"): value = value[1:-2] - if not set(value) <= set(("0", "1")): + if not frozenset(value) <= SET01: raise ValueError("B's coding contains unacceptable chars") return self._bits2octets(value) - elif value.endswith("'H"): + if value.endswith("'H"): value = value[1:-2] return ( len(value) * 4, @@ -2052,8 +2318,7 @@ class BitString(Obj): ) if isinstance(value, binary_type): return (len(value) * 8, value) - else: - raise InvalidValueType((self.__class__, string_types, binary_type)) + raise InvalidValueType((self.__class__, string_types, binary_type)) if isinstance(value, tuple): if ( len(value) == 2 and @@ -2069,11 +2334,13 @@ class BitString(Obj): bits.append(bit) if len(bits) == 0: return self._bits2octets("") - bits = set(bits) + bits = frozenset(bits) return self._bits2octets("".join( ("1" if bit in bits else "0") for bit in six_xrange(max(bits) + 1) )) + if issubclass(value.__class__, BitString): + return value._value raise InvalidValueType((self.__class__, binary_type, string_types)) @property @@ -2093,6 +2360,9 @@ class BitString(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __iter__(self): @@ -2122,7 +2392,7 @@ class BitString(Obj): @property def named(self): - return [name for name, bit in self.specs.items() if self[bit]] + return [name for name, bit in iteritems(self.specs) if self[bit]] def __call__( self, @@ -2236,8 +2506,8 @@ class BitString(Obj): offset=offset, ) if t == self.tag: - if tag_only: - return + if tag_only: # pragma: no cover + return None return self._decode_chunk(lv, offset, decode_path, ctx) if t == self.tag_constructed: if not ctx.get("bered", False): @@ -2247,8 +2517,8 @@ class BitString(Obj): decode_path=decode_path, offset=offset, ) - if tag_only: - return + if tag_only: # pragma: no cover + return None lenindef = False try: l, llen, v = len_decode(lv) @@ -2301,6 +2571,7 @@ class BitString(Obj): decode_path=sub_decode_path, leavemm=True, ctx=ctx, + _ctx_immutable=False, ) except TagMismatch: raise DecodeError( @@ -2345,7 +2616,7 @@ class BitString(Obj): _decoded=(offset, llen, vlen + (EOC_LEN if lenindef else 0)), ) obj.lenindef = lenindef - obj.bered = True + obj.ber_encoded = True return obj, (v[EOC_LEN:] if lenindef else v) raise TagMismatch( klass=self.__class__, @@ -2365,6 +2636,7 @@ class BitString(Obj): if len(self.specs) > 0: blob = tuple(self.named) yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -2384,6 +2656,7 @@ class BitString(Obj): expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, lenindef=self.lenindef, + ber_encoded=self.ber_encoded, bered=self.bered, ) defined_by, defined = self.defined or (None, None) @@ -2476,10 +2749,10 @@ class OctetString(Obj): ) def _value_sanitize(self, value): - if issubclass(value.__class__, OctetString): - value = value._value - elif isinstance(value, binary_type): + if isinstance(value, binary_type): pass + elif issubclass(value.__class__, OctetString): + value = value._value else: raise InvalidValueType((self.__class__, bytes)) if not self._bound_min <= len(value) <= self._bound_max: @@ -2502,6 +2775,9 @@ class OctetString(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __bytes__(self): @@ -2607,7 +2883,7 @@ class OctetString(Obj): ) if t == self.tag: if tag_only: - return + return None return self._decode_chunk(lv, offset, decode_path, ctx) if t == self.tag_constructed: if not ctx.get("bered", False): @@ -2618,7 +2894,7 @@ class OctetString(Obj): offset=offset, ) if tag_only: - return + return None lenindef = False try: l, llen, v = len_decode(lv) @@ -2664,6 +2940,7 @@ class OctetString(Obj): decode_path=sub_decode_path, leavemm=True, ctx=ctx, + _ctx_immutable=False, ) except TagMismatch: raise DecodeError( @@ -2701,7 +2978,7 @@ class OctetString(Obj): offset=offset, ) obj.lenindef = lenindef - obj.bered = True + obj.ber_encoded = True return obj, (v[EOC_LEN:] if lenindef else v) raise TagMismatch( klass=self.__class__, @@ -2714,6 +2991,7 @@ class OctetString(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -2733,6 +3011,7 @@ class OctetString(Obj): expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, lenindef=self.lenindef, + ber_encoded=self.ber_encoded, bered=self.bered, ) defined_by, defined = self.defined or (None, None) @@ -2785,6 +3064,9 @@ class Null(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __eq__(self, their): @@ -2827,8 +3109,8 @@ class Null(Obj): decode_path=decode_path, offset=offset, ) - if tag_only: - return + if tag_only: # pragma: no cover + return None try: l, _, v = len_decode(lv) except DecodeError as err: @@ -2858,6 +3140,7 @@ class Null(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -2873,6 +3156,7 @@ class Null(Obj): expl_llen=self.expl_llen if self.expled else None, expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, + bered=self.bered, ) for pp in self.pps_lenindef(decode_path): yield pp @@ -2993,6 +3277,9 @@ class ObjectIdentifier(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __iter__(self): @@ -3077,8 +3364,8 @@ class ObjectIdentifier(Obj): decode_path=decode_path, offset=offset, ) - if tag_only: - return + if tag_only: # pragma: no cover + return None try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -3104,11 +3391,17 @@ class ObjectIdentifier(Obj): ) v, tail = v[:l], v[l:] arcs = [] + ber_encoded = False while len(v) > 0: i = 0 arc = 0 while True: octet = indexbytes(v, i) + if i == 0 and octet == 0x80: + if ctx.get("bered", False): + ber_encoded = True + else: + raise DecodeError("non normalized arc encoding") arc = (arc << 7) | (octet & 0x7F) if octet & 0x80 == 0: arcs.append(arc) @@ -3140,6 +3433,8 @@ class ObjectIdentifier(Obj): optional=self.optional, _decoded=(offset, llen, l), ) + if ber_encoded: + obj.ber_encoded = True return obj, tail def __repr__(self): @@ -3147,6 +3442,7 @@ class ObjectIdentifier(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -3164,6 +3460,8 @@ class ObjectIdentifier(Obj): expl_llen=self.expl_llen if self.expled else None, expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, + ber_encoded=self.ber_encoded, + bered=self.bered, ) for pp in self.pps_lenindef(decode_path): yield pp @@ -3206,7 +3504,10 @@ class Enumerated(Integer): if isinstance(value, self.__class__): value = value._value elif isinstance(value, integer_types): - if value not in list(self.specs.values()): + for _value in itervalues(self.specs): + if _value == value: + break + else: raise DecodeError( "unknown integer value: %s" % value, klass=self.__class__, @@ -3231,6 +3532,9 @@ class Enumerated(Integer): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __call__( @@ -3372,6 +3676,7 @@ class CommonString(OctetString): if self.ready: value = hexenc(bytes(self)) if no_unicode else self.__unicode__() yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -3389,6 +3694,8 @@ class CommonString(OctetString): expl_llen=self.expl_llen if self.expled else None, expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, + ber_encoded=self.ber_encoded, + bered=self.bered, ) for pp in self.pps_lenindef(decode_path): yield pp @@ -3401,29 +3708,57 @@ class UTF8String(CommonString): asn1_type_name = "UTF8String" -class NumericString(CommonString): +class AllowableCharsMixin(object): + @property + def allowable_chars(self): + if PY2: + return self._allowable_chars + return frozenset(six_unichr(c) for c in self._allowable_chars) + + +class NumericString(AllowableCharsMixin, CommonString): """Numeric string - Its value is properly sanitized: only ASCII digits can be stored. + Its value is properly sanitized: only ASCII digits with spaces can + be stored. + + >>> NumericString().allowable_chars + frozenset(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ' ']) """ __slots__ = () tag_default = tag_encode(18) encoding = "ascii" asn1_type_name = "NumericString" - allowable_chars = set(digits.encode("ascii")) + _allowable_chars = frozenset(digits.encode("ascii") + b" ") def _value_sanitize(self, value): value = super(NumericString, self)._value_sanitize(value) - if not set(value) <= self.allowable_chars: + if not frozenset(value) <= self._allowable_chars: raise DecodeError("non-numeric value") return value -class PrintableString(CommonString): +class PrintableString(AllowableCharsMixin, CommonString): + """Printable string + + Its value is properly sanitized: see X.680 41.4 table 10. + + >>> PrintableString().allowable_chars + frozenset([' ', "'", ..., 'z']) + """ __slots__ = () tag_default = tag_encode(19) encoding = "ascii" asn1_type_name = "PrintableString" + _allowable_chars = frozenset( + (ascii_letters + digits + " '()+,-./:=?").encode("ascii") + ) + + def _value_sanitize(self, value): + value = super(PrintableString, self)._value_sanitize(value) + if not frozenset(value) <= self._allowable_chars: + raise DecodeError("non-printable value") + return value class TeletexString(CommonString): @@ -3470,14 +3805,16 @@ class UTCTime(CommonString): datetime.datetime(2017, 9, 30, 22, 7, 50) >>> UTCTime(datetime(2057, 9, 30, 22, 7, 50)).todatetime() datetime.datetime(1957, 9, 30, 22, 7, 50) + + .. warning:: + + BER encoding is unsupported. """ __slots__ = () tag_default = tag_encode(23) encoding = "ascii" asn1_type_name = "UTCTime" - fmt = "%y%m%d%H%M%SZ" - def __init__( self, value=None, @@ -3516,21 +3853,36 @@ class UTCTime(CommonString): if self._value is None: self._value = default + def _strptime(self, value): + # datetime.strptime's format: %y%m%d%H%M%SZ + if len(value) != LEN_YYMMDDHHMMSSZ: + raise ValueError("invalid UTCTime length") + if value[-1] != "Z": + raise ValueError("non UTC timezone") + return datetime( + 2000 + int(value[:2]), # %y + int(value[2:4]), # %m + int(value[4:6]), # %d + int(value[6:8]), # %H + int(value[8:10]), # %M + int(value[10:12]), # %S + ) + def _value_sanitize(self, value): + if isinstance(value, binary_type): + try: + value_decoded = value.decode("ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as err: + raise DecodeError("invalid UTCTime encoding: %r" % err) + try: + self._strptime(value_decoded) + except (TypeError, ValueError) as err: + raise DecodeError("invalid UTCTime format: %r" % err) + return value if isinstance(value, self.__class__): return value._value if isinstance(value, datetime): - return value.strftime(self.fmt).encode("ascii") - if isinstance(value, binary_type): - value_decoded = value.decode("ascii") - if len(value_decoded) == LEN_YYMMDDHHMMSSZ: - try: - datetime.strptime(value_decoded, self.fmt) - except ValueError: - raise DecodeError("invalid UTCTime format") - return value - else: - raise DecodeError("invalid UTCTime length") + return value.strftime("%y%m%d%H%M%SZ").encode("ascii") raise InvalidValueType((self.__class__, datetime)) def __eq__(self, their): @@ -3555,7 +3907,7 @@ class UTCTime(CommonString): having < 50 years are treated as 20xx, 19xx otherwise, according to X.509 recomendation. """ - value = datetime.strptime(self._value.decode("ascii"), self.fmt) + value = self._strptime(self._value.decode("ascii")) year = value.year % 100 return datetime( year=(2000 + year) if year < 50 else (1900 + year), @@ -3571,6 +3923,7 @@ class UTCTime(CommonString): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -3588,6 +3941,8 @@ class UTCTime(CommonString): expl_llen=self.expl_llen if self.expled else None, expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, + ber_encoded=self.ber_encoded, + bered=self.bered, ) for pp in self.pps_lenindef(decode_path): yield pp @@ -3604,51 +3959,85 @@ class GeneralizedTime(UTCTime): '20170930220750.000123Z' >>> t = GeneralizedTime(datetime(2057, 9, 30, 22, 7, 50)) GeneralizedTime GeneralizedTime 2057-09-30T22:07:50 + + .. warning:: + + BER encoding is unsupported. + + .. warning:: + + Only microsecond fractions are supported. + :py:exc:`pyderasn.DecodeError` will be raised during decoding of + higher precision values. """ __slots__ = () tag_default = tag_encode(24) asn1_type_name = "GeneralizedTime" - fmt = "%Y%m%d%H%M%SZ" - fmt_ms = "%Y%m%d%H%M%S.%fZ" + def _strptime(self, value): + l = len(value) + if l == LEN_YYYYMMDDHHMMSSZ: + # datetime.strptime's format: %y%m%d%H%M%SZ + if value[-1] != "Z": + raise ValueError("non UTC timezone") + return datetime( + int(value[:4]), # %Y + int(value[4:6]), # %m + int(value[6:8]), # %d + int(value[8:10]), # %H + int(value[10:12]), # %M + int(value[12:14]), # %S + ) + if l >= LEN_YYYYMMDDHHMMSSDMZ: + # datetime.strptime's format: %Y%m%d%H%M%S.%fZ + if value[-1] != "Z": + raise ValueError("non UTC timezone") + if value[14] != ".": + raise ValueError("no fractions separator") + us = value[15:-1] + if us[-1] == "0": + raise ValueError("trailing zero") + us_len = len(us) + if us_len > 6: + raise ValueError("only microsecond fractions are supported") + us = int(us + ("0" * (6 - us_len))) + decoded = datetime( + int(value[:4]), # %Y + int(value[4:6]), # %m + int(value[6:8]), # %d + int(value[8:10]), # %H + int(value[10:12]), # %M + int(value[12:14]), # %S + us, # %f + ) + return decoded + raise ValueError("invalid GeneralizedTime length") def _value_sanitize(self, value): - if isinstance(value, self.__class__): - return value._value - if isinstance(value, datetime): - return value.strftime( - self.fmt_ms if value.microsecond > 0 else self.fmt - ).encode("ascii") if isinstance(value, binary_type): - value_decoded = value.decode("ascii") - if len(value_decoded) == LEN_YYYYMMDDHHMMSSZ: - try: - datetime.strptime(value_decoded, self.fmt) - except ValueError: - raise DecodeError( - "invalid GeneralizedTime (without ms) format", - ) - return value - elif len(value_decoded) >= LEN_YYYYMMDDHHMMSSDMZ: - try: - datetime.strptime(value_decoded, self.fmt_ms) - except ValueError: - raise DecodeError( - "invalid GeneralizedTime (with ms) format", - ) - return value - else: + try: + value_decoded = value.decode("ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as err: + raise DecodeError("invalid GeneralizedTime encoding: %r" % err) + try: + self._strptime(value_decoded) + except (TypeError, ValueError) as err: raise DecodeError( - "invalid GeneralizedTime length", + "invalid GeneralizedTime format: %r" % err, klass=self.__class__, ) + return value + if isinstance(value, self.__class__): + return value._value + if isinstance(value, datetime): + encoded = value.strftime("%Y%m%d%H%M%S") + if value.microsecond > 0: + encoded = encoded + (".%06d" % value.microsecond).rstrip("0") + return (encoded + "Z").encode("ascii") raise InvalidValueType((self.__class__, datetime)) def todatetime(self): - value = self._value.decode("ascii") - if len(value) == LEN_YYYYMMDDHHMMSSZ: - return datetime.strptime(value, self.fmt) - return datetime.strptime(value, self.fmt_ms) + return self._strptime(self._value.decode("ascii")) class GraphicString(CommonString): @@ -3767,8 +4156,6 @@ class Choice(Obj): self._value = default_obj.copy()._value def _value_sanitize(self, value): - if isinstance(value, self.__class__): - return value._value if isinstance(value, tuple) and len(value) == 2: choice, obj = value spec = self.specs.get(choice) @@ -3777,12 +4164,21 @@ class Choice(Obj): if not isinstance(obj, spec.__class__): raise InvalidValueType((spec,)) return (choice, spec(obj)) + if isinstance(value, self.__class__): + return value._value raise InvalidValueType((self.__class__, tuple)) @property def ready(self): return self._value is not None and self._value[1].ready + @property + def bered(self): + return self.expl_lenindef or ( + (self._value is not None) and + self._value[1].bered + ) + def copy(self): obj = self.__class__(schema=self.specs) obj._expl = self._expl @@ -3791,6 +4187,9 @@ class Choice(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded value = self._value if value is not None: obj._value = (value[0], value[1].copy()) @@ -3862,7 +4261,7 @@ class Choice(Obj): return self._value[1].encode() def _decode(self, tlv, offset, decode_path, ctx, tag_only): - for choice, spec in self.specs.items(): + for choice, spec in iteritems(self.specs): sub_decode_path = decode_path + (choice,) try: spec.decode( @@ -3872,6 +4271,7 @@ class Choice(Obj): decode_path=sub_decode_path, ctx=ctx, tag_only=True, + _ctx_immutable=False, ) except TagMismatch: continue @@ -3882,21 +4282,22 @@ class Choice(Obj): decode_path=decode_path, offset=offset, ) - if tag_only: - return + if tag_only: # pragma: no cover + return None value, tail = spec.decode( tlv, offset=offset, leavemm=True, decode_path=sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) obj = self.__class__( schema=self.specs, expl=self._expl, default=self.default, optional=self.optional, - _decoded=(offset, 0, value.tlvlen), + _decoded=(offset, 0, value.fulllen), ) obj._value = (choice, value) return obj, tail @@ -3909,6 +4310,7 @@ class Choice(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -3922,6 +4324,7 @@ class Choice(Obj): llen=self.llen, vlen=self.vlen, expl_lenindef=self.expl_lenindef, + bered=self.bered, ) if self.ready: yield self.value.pps(decode_path=decode_path + (self.choice,)) @@ -3998,18 +4401,26 @@ class Any(Obj): self.defined = None def _value_sanitize(self, value): + if isinstance(value, binary_type): + return value if isinstance(value, self.__class__): return value._value if isinstance(value, Obj): return value.encode() - if isinstance(value, binary_type): - return value raise InvalidValueType((self.__class__, Obj, binary_type)) @property def ready(self): return self._value is not None + @property + def bered(self): + if self.expl_lenindef or self.lenindef: + return True + if self.defined is None: + return False + return self.defined[1].bered + def copy(self): obj = self.__class__() obj._value = self._value @@ -4019,6 +4430,9 @@ class Any(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __eq__(self, their): @@ -4082,6 +4496,7 @@ class Any(Obj): decode_path=decode_path + (str(chunk_i),), leavemm=True, ctx=ctx, + _ctx_immutable=False, ) vlen += chunk.tlvlen sub_offset += chunk.tlvlen @@ -4126,6 +4541,7 @@ class Any(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -4144,6 +4560,7 @@ class Any(Obj): expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, lenindef=self.lenindef, + bered=self.bered, ) defined_by, defined = self.defined or (None, None) if defined_by is not None: @@ -4174,8 +4591,7 @@ def get_def_by_path(defines_by_path, sub_decode_path): def abs_decode_path(decode_path, rel_path): """Create an absolute decode path from current and relative ones - :param decode_path: current decode path, starting point. - Tuple of strings + :param decode_path: current decode path, starting point. Tuple of strings :param rel_path: relative path to ``decode_path``. Tuple of strings. If first tuple's element is "/", then treat it as an absolute path, ignoring ``decode_path`` as @@ -4276,18 +4692,14 @@ class Sequence(Obj): All defaulted values are always optional. - .. _strict_default_existence_ctx: + .. _allow_default_values_ctx: - .. warning:: - - When decoded DER contains defaulted value inside, then - technically this is not valid DER encoding. But we allow and pass - it **by default**. Of course reencoding of that kind of DER will - result in different binary representation (validly without - defaulted value inside). You can enable strict defaulted values - existence validation by setting ``"strict_default_existence": - True`` :ref:`context ` option -- decoding process will raise - an exception if defaulted value is met. + DER prohibits default value encoding and will raise an error if + default value is unexpectedly met during decode. + If :ref:`bered ` context option is set, then no error + will be raised, but ``bered`` attribute set. You can disable strict + defaulted values existence validation by setting + ``"allow_default_values": True`` :ref:`context ` option. Two sequences are equal if they have equal specification (schema), implicit/explicit tagging and the same values. @@ -4334,17 +4746,22 @@ class Sequence(Obj): @property def ready(self): - for name, spec in self.specs.items(): + for name, spec in iteritems(self.specs): value = self._value.get(name) if value is None: if spec.optional: continue return False - else: - if not value.ready: - return False + if not value.ready: + return False return True + @property + def bered(self): + if self.expl_lenindef or self.lenindef or self.ber_encoded: + return True + return any(value.bered for value in itervalues(self._value)) + def copy(self): obj = self.__class__(schema=self.specs) obj.tag = self.tag @@ -4354,7 +4771,10 @@ class Sequence(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen - obj._value = {k: v.copy() for k, v in self._value.items()} + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded + obj._value = {k: v.copy() for k, v in iteritems(self._value)} return obj def __eq__(self, their): @@ -4415,7 +4835,7 @@ class Sequence(Obj): def _encoded_values(self): raws = [] - for name, spec in self.specs.items(): + for name, spec in iteritems(self.specs): value = self._value.get(name) if value is None: if spec.optional: @@ -4444,13 +4864,14 @@ class Sequence(Obj): decode_path=decode_path, offset=offset, ) - if tag_only: - return + if tag_only: # pragma: no cover + return None lenindef = False + ctx_bered = ctx.get("bered", False) try: l, llen, v = len_decode(lv) except LenIndefForm as err: - if not ctx.get("bered", False): + if not ctx_bered: raise err.__class__( msg=err.msg, klass=self.__class__, @@ -4478,7 +4899,9 @@ class Sequence(Obj): vlen = 0 sub_offset = offset + tlen + llen values = {} - for name, spec in self.specs.items(): + ber_encoded = False + ctx_allow_default_values = ctx.get("allow_default_values", False) + for name, spec in iteritems(self.specs): if spec.optional and ( (lenindef and v[:EOC_LEN].tobytes() == EOC) or len(v) == 0 @@ -4492,13 +4915,14 @@ class Sequence(Obj): leavemm=True, decode_path=sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) - except TagMismatch: - if spec.optional: + except TagMismatch as err: + if (len(err.decode_path) == len(decode_path) + 1) and spec.optional: continue raise - defined = get_def_by_path(ctx.get("defines", ()), sub_decode_path) + defined = get_def_by_path(ctx.get("_defines", ()), sub_decode_path) if defined is not None: defined_by, defined_spec = defined if issubclass(value.__class__, SequenceOf): @@ -4516,6 +4940,7 @@ class Sequence(Obj): leavemm=True, decode_path=sub_sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) if len(defined_tail) > 0: raise DecodeError( @@ -4535,6 +4960,7 @@ class Sequence(Obj): leavemm=True, decode_path=sub_decode_path + (DecodePathDefBy(defined_by),), ctx=ctx, + _ctx_immutable=False, ) if len(defined_tail) > 0: raise DecodeError( @@ -4550,15 +4976,15 @@ class Sequence(Obj): sub_offset += value_len v = v_tail if spec.default is not None and value == spec.default: - if ctx.get("strict_default_existence", False): + if ctx_bered or ctx_allow_default_values: + ber_encoded = True + else: raise DecodeError( "DEFAULT value met", klass=self.__class__, decode_path=sub_decode_path, offset=sub_offset, ) - else: - continue values[name] = value spec_defines = getattr(spec, "defines", ()) @@ -4570,7 +4996,7 @@ class Sequence(Obj): for rel_path, schema in spec_defines: defined = schema.get(value, None) if defined is not None: - ctx.setdefault("defines", []).append(( + ctx.setdefault("_defines", []).append(( abs_decode_path(sub_decode_path[:-1], rel_path), (value, defined), )) @@ -4601,6 +5027,7 @@ class Sequence(Obj): ) obj._value = values obj.lenindef = lenindef + obj.ber_encoded = ber_encoded return obj, tail def __repr__(self): @@ -4615,6 +5042,7 @@ class Sequence(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -4632,6 +5060,8 @@ class Sequence(Obj): expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, lenindef=self.lenindef, + ber_encoded=self.ber_encoded, + bered=self.bered, ) for name in self.specs: value = self._value.get(name) @@ -4646,6 +5076,14 @@ class Set(Sequence): """``SET`` structure type Its usage is identical to :py:class:`pyderasn.Sequence`. + + .. _allow_unordered_set_ctx: + + DER prohibits unordered values encoding and will raise an error + during decode. If If :ref:`bered ` context option is set, + then no error will occure. Also you can disable strict values + ordering check by setting ``"allow_unordered_set": True`` + :ref:`context ` option. """ __slots__ = () tag_default = tag_encode(form=TagFormConstructed, num=17) @@ -4657,6 +5095,9 @@ class Set(Sequence): v = b"".join(raws) return b"".join((self.tag, len_encode(len(v)), v)) + def _specs_items(self): + return iteritems(self.specs) + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, tlen, lv = tag_strip(tlv) @@ -4674,12 +5115,13 @@ class Set(Sequence): offset=offset, ) if tag_only: - return + return None lenindef = False + ctx_bered = ctx.get("bered", False) try: l, llen, v = len_decode(lv) except LenIndefForm as err: - if not ctx.get("bered", False): + if not ctx_bered: raise err.__class__( msg=err.msg, klass=self.__class__, @@ -4706,11 +5148,15 @@ class Set(Sequence): vlen = 0 sub_offset = offset + tlen + llen values = {} - specs_items = self.specs.items + ber_encoded = False + ctx_allow_default_values = ctx.get("allow_default_values", False) + ctx_allow_unordered_set = ctx.get("allow_unordered_set", False) + value_prev = memoryview(v[:0]) + while len(v) > 0: if lenindef and v[:EOC_LEN].tobytes() == EOC: break - for name, spec in specs_items(): + for name, spec in self._specs_items(): sub_decode_path = decode_path + (name,) try: spec.decode( @@ -4720,6 +5166,7 @@ class Set(Sequence): decode_path=sub_decode_path, ctx=ctx, tag_only=True, + _ctx_immutable=False, ) except TagMismatch: continue @@ -4736,14 +5183,35 @@ class Set(Sequence): leavemm=True, decode_path=sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) value_len = value.fulllen + if value_prev.tobytes() > v[:value_len].tobytes(): + if ctx_bered or ctx_allow_unordered_set: + ber_encoded = True + else: + raise DecodeError( + "unordered " + self.asn1_type_name, + klass=self.__class__, + decode_path=sub_decode_path, + offset=sub_offset, + ) + if spec.default is None or value != spec.default: + pass + elif ctx_bered or ctx_allow_default_values: + ber_encoded = True + else: + raise DecodeError( + "DEFAULT value met", + klass=self.__class__, + decode_path=sub_decode_path, + offset=sub_offset, + ) + values[name] = value + value_prev = v[:value_len] sub_offset += value_len vlen += value_len v = v_tail - if spec.default is None or value != spec.default: # pragma: no cover - # SeqMixing.test_encoded_default_accepted covers that place - values[name] = value obj = self.__class__( schema=self.specs, impl=self.tag, @@ -4752,7 +5220,6 @@ class Set(Sequence): optional=self.optional, _decoded=(offset, llen, vlen + (EOC_LEN if lenindef else 0)), ) - obj._value = values if lenindef: if v[:EOC_LEN].tobytes() != EOC: raise DecodeError( @@ -4763,6 +5230,7 @@ class Set(Sequence): ) tail = v[EOC_LEN:] obj.lenindef = True + obj._value = values if not obj.ready: raise DecodeError( "not all values are ready", @@ -4770,6 +5238,7 @@ class Set(Sequence): decode_path=decode_path, offset=offset, ) + obj.ber_encoded = ber_encoded return obj, tail @@ -4868,6 +5337,12 @@ class SequenceOf(Obj): def ready(self): return all(v.ready for v in self._value) + @property + def bered(self): + if self.expl_lenindef or self.lenindef or self.ber_encoded: + return True + return any(v.bered for v in self._value) + def copy(self): obj = self.__class__(schema=self.spec) obj._bound_min = self._bound_min @@ -4879,6 +5354,9 @@ class SequenceOf(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded obj._value = [v.copy() for v in self._value] return obj @@ -4953,7 +5431,7 @@ class SequenceOf(Obj): v = b"".join(self._encoded_values()) return b"".join((self.tag, len_encode(len(v)), v)) - def _decode(self, tlv, offset, decode_path, ctx, tag_only): + def _decode(self, tlv, offset, decode_path, ctx, tag_only, ordering_check=False): try: t, tlen, lv = tag_strip(tlv) except DecodeError as err: @@ -4970,12 +5448,13 @@ class SequenceOf(Obj): offset=offset, ) if tag_only: - return + return None lenindef = False + ctx_bered = ctx.get("bered", False) try: l, llen, v = len_decode(lv) except LenIndefForm as err: - if not ctx.get("bered", False): + if not ctx_bered: raise err.__class__( msg=err.msg, klass=self.__class__, @@ -5003,32 +5482,57 @@ class SequenceOf(Obj): vlen = 0 sub_offset = offset + tlen + llen _value = [] + ctx_allow_unordered_set = ctx.get("allow_unordered_set", False) + value_prev = memoryview(v[:0]) + ber_encoded = False spec = self.spec while len(v) > 0: if lenindef and v[:EOC_LEN].tobytes() == EOC: break + sub_decode_path = decode_path + (str(len(_value)),) value, v_tail = spec.decode( v, sub_offset, leavemm=True, - decode_path=decode_path + (str(len(_value)),), + decode_path=sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) value_len = value.fulllen + if ordering_check: + if value_prev.tobytes() > v[:value_len].tobytes(): + if ctx_bered or ctx_allow_unordered_set: + ber_encoded = True + else: + raise DecodeError( + "unordered " + self.asn1_type_name, + klass=self.__class__, + decode_path=sub_decode_path, + offset=sub_offset, + ) + value_prev = v[:value_len] + _value.append(value) sub_offset += value_len vlen += value_len v = v_tail - _value.append(value) - obj = self.__class__( - value=_value, - schema=spec, - bounds=(self._bound_min, self._bound_max), - impl=self.tag, - expl=self._expl, - default=self.default, - optional=self.optional, - _decoded=(offset, llen, vlen + (EOC_LEN if lenindef else 0)), - ) + try: + obj = self.__class__( + value=_value, + schema=spec, + bounds=(self._bound_min, self._bound_max), + impl=self.tag, + expl=self._expl, + default=self.default, + optional=self.optional, + _decoded=(offset, llen, vlen + (EOC_LEN if lenindef else 0)), + ) + except BoundsError as err: + raise DecodeError( + msg=str(err), + klass=self.__class__, + decode_path=decode_path, + offset=offset, + ) if lenindef: if v[:EOC_LEN].tobytes() != EOC: raise DecodeError( @@ -5039,6 +5543,7 @@ class SequenceOf(Obj): ) obj.lenindef = True tail = v[EOC_LEN:] + obj.ber_encoded = ber_encoded return obj, tail def __repr__(self): @@ -5049,6 +5554,7 @@ class SequenceOf(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -5066,6 +5572,8 @@ class SequenceOf(Obj): expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, lenindef=self.lenindef, + ber_encoded=self.ber_encoded, + bered=self.bered, ) for i, value in enumerate(self._value): yield value.pps(decode_path=decode_path + (str(i),)) @@ -5088,6 +5596,16 @@ class SetOf(SequenceOf): v = b"".join(raws) return b"".join((self.tag, len_encode(len(v)), v)) + def _decode(self, tlv, offset, decode_path, ctx, tag_only): + return super(SetOf, self)._decode( + tlv, + offset, + decode_path, + ctx, + tag_only, + ordering_check=True, + ) + def obj_by_path(pypath): # pragma: no cover """Import object specified as string Python path @@ -5112,7 +5630,7 @@ def generic_decoder(): # pragma: no cover choice = PrimitiveTypes() choice.specs["SequenceOf"] = SequenceOf(schema=choice) choice.specs["SetOf"] = SetOf(schema=choice) - for i in range(31): + for i in six_xrange(31): choice.specs["SequenceOf%d" % i] = SequenceOf( schema=choice, expl=tag_ctxc(i), @@ -5124,10 +5642,21 @@ def generic_decoder(): # pragma: no cover __slots__ = () schema = choice - def pprint_any(obj, oids=None, with_colours=False): + def pprint_any( + obj, + oid_maps=(), + with_colours=False, + with_decode_path=False, + decode_path_only=(), + ): def _pprint_pps(pps): for pp in pps: if hasattr(pp, "_fields"): + if ( + decode_path_only != () and + pp.decode_path[:len(decode_path_only)] != decode_path_only + ): + continue if pp.asn1_type_name == Choice.asn1_type_name: continue pp_kwargs = pp._asdict() @@ -5135,12 +5664,17 @@ def generic_decoder(): # pragma: no cover pp = _pp(**pp_kwargs) yield pp_console_row( pp, - oids=oids, + oid_maps=oid_maps, with_offsets=True, with_blob=False, with_colours=with_colours, + with_decode_path=with_decode_path, + decode_path_len_decrease=len(decode_path_only), ) - for row in pp_console_blob(pp): + for row in pp_console_blob( + pp, + decode_path_len_decrease=len(decode_path_only), + ): yield row else: for row in _pprint_pps(pp): @@ -5160,7 +5694,7 @@ def main(): # pragma: no cover ) parser.add_argument( "--oids", - help="Python path to dictionary with OIDs", + help="Python paths to dictionary with OIDs, comma separated", ) parser.add_argument( "--schema", @@ -5172,9 +5706,23 @@ def main(): # pragma: no cover ) parser.add_argument( "--nobered", - action='store_true', + action="store_true", help="Disallow BER encoding", ) + parser.add_argument( + "--print-decode-path", + action="store_true", + help="Print decode paths", + ) + parser.add_argument( + "--decode-path-only", + help="Print only specified decode path", + ) + parser.add_argument( + "--allow-expl-oob", + action="store_true", + help="Allow explicit tag out-of-bound", + ) parser.add_argument( "DERFile", type=argparse.FileType("rb"), @@ -5184,21 +5732,32 @@ def main(): # pragma: no cover args.DERFile.seek(args.skip) der = memoryview(args.DERFile.read()) args.DERFile.close() - oids = obj_by_path(args.oids) if args.oids else {} + oid_maps = ( + [obj_by_path(_path) for _path in (args.oids or "").split(",")] + if args.oids else () + ) if args.schema: schema = obj_by_path(args.schema) from functools import partial pprinter = partial(pprint, big_blobs=True) else: schema, pprinter = generic_decoder() - ctx = {"bered": not args.nobered} + ctx = { + "bered": not args.nobered, + "allow_expl_oob": args.allow_expl_oob, + } if args.defines_by_path is not None: ctx["defines_by_path"] = obj_by_path(args.defines_by_path) obj, tail = schema().decode(der, ctx=ctx) print(pprinter( obj, - oids=oids, + oid_maps=oid_maps, with_colours=True if environ.get("NO_COLOR") is None else False, + with_decode_path=args.print_decode_path, + decode_path_only=( + () if args.decode_path_only is None else + tuple(args.decode_path_only.split(":")) + ), )) if tail != b"": print("\nTrailing data: %s" % hexenc(tail))