X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=pyderasn.py;h=50faa0be870ced74ab9d33afb4ac0f4008846421;hb=97fe8b8eae995499566e3ae26e2e941e633997bc;hp=aa91b04052c100a8ba6dfc102cd840dcfce64c4a;hpb=31fe69135e96d6cca5410168e92abfc55eda5197;p=pyderasn.git diff --git a/pyderasn.py b/pyderasn.py index aa91b04..50faa0b 100755 --- a/pyderasn.py +++ b/pyderasn.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding: utf-8 # PyDERASN -- Python ASN.1 DER/BER codec with abstract structures -# Copyright (C) 2017-2018 Sergey Matveev +# Copyright (C) 2017-2019 Sergey Matveev # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as @@ -384,7 +384,8 @@ constructed primitive types should be parsed successfully. * If object is encoded in BER form (not the DER one), then ``ber_encoded`` attribute is set to True. Only ``BOOLEAN``, ``BIT STRING``, ``OCTET - STRING``, ``SEQUENCE``, ``SET``, ``SET OF`` can contain it. + STRING``, ``OBJECT IDENTIFIER``, ``SEQUENCE``, ``SET``, ``SET OF`` + can contain it. * If object has an indefinite length encoding, then its ``lenindef`` attribute is set to True. Only ``BIT STRING``, ``OCTET STRING``, ``SEQUENCE``, ``SET``, ``SEQUENCE OF``, ``SET OF``, ``ANY`` can @@ -515,6 +516,7 @@ Various ------- .. autofunction:: pyderasn.abs_decode_path +.. autofunction:: pyderasn.colonize_hex .. autofunction:: pyderasn.hexenc .. autofunction:: pyderasn.hexdec .. autofunction:: pyderasn.tag_encode @@ -539,6 +541,7 @@ from codecs import getdecoder from codecs import getencoder from collections import namedtuple from collections import OrderedDict +from copy import copy from datetime import datetime from math import ceil from os import environ @@ -552,6 +555,8 @@ from six import indexbytes from six import int2byte from six import integer_types from six import iterbytes +from six import iteritems +from six import itervalues from six import PY2 from six import string_types from six import text_type @@ -646,6 +651,7 @@ LENINDEF_PP_CHAR = "I" if PY2 else "∞" class ASN1Error(ValueError): pass + class DecodeError(ASN1Error): def __init__(self, msg="", klass=None, decode_path=(), offset=0): """ @@ -1027,6 +1033,7 @@ class Obj(object): decode_path=(), ctx=None, tag_only=False, + _ctx_immutable=True, ): """Decode the data @@ -1038,10 +1045,13 @@ class Obj(object): :param tag_only: decode only the tag, without length and contents (used only in Choice and Set structures, trying to determine if tag satisfies the scheme) + :param _ctx_immutable: do we need to copy ``ctx`` before using it :returns: (Obj, remaining data) """ if ctx is None: ctx = {} + elif _ctx_immutable: + ctx = copy(ctx) tlv = memoryview(data) if self._expl is None: result = self._decode( @@ -1235,6 +1245,7 @@ class DecodePathDefBy(object): ######################################################################## PP = namedtuple("PP", ( + "obj", "asn1_type_name", "obj_name", "decode_path", @@ -1260,6 +1271,7 @@ PP = namedtuple("PP", ( def _pp( + obj=None, asn1_type_name="unknown", obj_name="unknown", decode_path=(), @@ -1283,6 +1295,7 @@ def _pp( bered=False, ): return PP( + obj, asn1_type_name, obj_name, decode_path, @@ -1311,6 +1324,12 @@ def _colourize(what, colour, with_colours, attrs=("bold",)): return colored(what, colour, attrs=attrs) if with_colours else what +def colonize_hex(hexed): + """Separate hexadecimal string with colons + """ + return ":".join(hexed[i:i + 2] for i in six_xrange(0, len(hexed), 2)) + + def pp_console_row( pp, oids=None, @@ -1381,6 +1400,15 @@ def pp_console_row( value in oids ): cols.append(_colourize("(%s)" % oids[value], "green", with_colours)) + if pp.asn1_type_name == Integer.asn1_type_name: + hex_repr = hex(int(pp.obj._value))[2:].upper() + if len(hex_repr) % 2 != 0: + hex_repr = "0" + hex_repr + cols.append(_colourize( + "(%s)" % colonize_hex(hex_repr), + "green", + with_colours, + )) if with_blob: if isinstance(pp.blob, binary_type): cols.append(hexenc(pp.blob)) @@ -1406,11 +1434,9 @@ def pp_console_blob(pp, decode_path_len_decrease=0): cols.append(" ." * (decode_path_len + 1)) if isinstance(pp.blob, binary_type): blob = hexenc(pp.blob).upper() - for i in range(0, len(blob), 32): + for i in six_xrange(0, len(blob), 32): chunk = blob[i:i + 32] - yield " ".join(cols + [":".join( - chunk[j:j + 2] for j in range(0, len(chunk), 2) - )]) + yield " ".join(cols + [colonize_hex(chunk)]) elif isinstance(pp.blob, tuple): yield " ".join(cols + [", ".join(pp.blob)]) @@ -1525,10 +1551,10 @@ class Boolean(Obj): self._value = default def _value_sanitize(self, value): - if issubclass(value.__class__, Boolean): - return value._value if isinstance(value, bool): return value + if issubclass(value.__class__, Boolean): + return value._value raise InvalidValueType((self.__class__, bool)) @property @@ -1545,6 +1571,9 @@ class Boolean(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __nonzero__(self): @@ -1663,6 +1692,7 @@ class Boolean(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -1770,10 +1800,10 @@ class Integer(Obj): self._value = default def _value_sanitize(self, value): - if issubclass(value.__class__, Integer): - value = value._value - elif isinstance(value, integer_types): + if isinstance(value, integer_types): pass + elif issubclass(value.__class__, Integer): + value = value._value elif isinstance(value, str): value = self.specs.get(value) if value is None: @@ -1800,6 +1830,9 @@ class Integer(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __int__(self): @@ -1830,7 +1863,7 @@ class Integer(Obj): @property def named(self): - for name, value in self.specs.items(): + for name, value in iteritems(self.specs): if value == self._value: return name @@ -1990,6 +2023,7 @@ class Integer(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -2013,6 +2047,9 @@ class Integer(Obj): yield pp +SET01 = frozenset(("0", "1")) + + class BitString(Obj): """``BIT STRING`` bit string type @@ -2118,8 +2155,6 @@ class BitString(Obj): return bit_len, bytes(octets) def _value_sanitize(self, value): - if issubclass(value.__class__, BitString): - return value._value if isinstance(value, (string_types, binary_type)): if ( isinstance(value, string_types) and @@ -2127,7 +2162,7 @@ class BitString(Obj): ): if value.endswith("'B"): value = value[1:-2] - if not set(value) <= set(("0", "1")): + if not frozenset(value) <= SET01: raise ValueError("B's coding contains unacceptable chars") return self._bits2octets(value) elif value.endswith("'H"): @@ -2155,11 +2190,13 @@ class BitString(Obj): bits.append(bit) if len(bits) == 0: return self._bits2octets("") - bits = set(bits) + bits = frozenset(bits) return self._bits2octets("".join( ("1" if bit in bits else "0") for bit in six_xrange(max(bits) + 1) )) + if issubclass(value.__class__, BitString): + return value._value raise InvalidValueType((self.__class__, binary_type, string_types)) @property @@ -2179,6 +2216,9 @@ class BitString(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __iter__(self): @@ -2208,7 +2248,7 @@ class BitString(Obj): @property def named(self): - return [name for name, bit in self.specs.items() if self[bit]] + return [name for name, bit in iteritems(self.specs) if self[bit]] def __call__( self, @@ -2387,6 +2427,7 @@ class BitString(Obj): decode_path=sub_decode_path, leavemm=True, ctx=ctx, + _ctx_immutable=False, ) except TagMismatch: raise DecodeError( @@ -2451,6 +2492,7 @@ class BitString(Obj): if len(self.specs) > 0: blob = tuple(self.named) yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -2563,10 +2605,10 @@ class OctetString(Obj): ) def _value_sanitize(self, value): - if issubclass(value.__class__, OctetString): - value = value._value - elif isinstance(value, binary_type): + if isinstance(value, binary_type): pass + elif issubclass(value.__class__, OctetString): + value = value._value else: raise InvalidValueType((self.__class__, bytes)) if not self._bound_min <= len(value) <= self._bound_max: @@ -2589,6 +2631,9 @@ class OctetString(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __bytes__(self): @@ -2751,6 +2796,7 @@ class OctetString(Obj): decode_path=sub_decode_path, leavemm=True, ctx=ctx, + _ctx_immutable=False, ) except TagMismatch: raise DecodeError( @@ -2801,6 +2847,7 @@ class OctetString(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -2873,6 +2920,9 @@ class Null(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __eq__(self, their): @@ -2946,6 +2996,7 @@ class Null(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -3082,6 +3133,9 @@ class ObjectIdentifier(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __iter__(self): @@ -3193,11 +3247,17 @@ class ObjectIdentifier(Obj): ) v, tail = v[:l], v[l:] arcs = [] + ber_encoded = False while len(v) > 0: i = 0 arc = 0 while True: octet = indexbytes(v, i) + if i == 0 and octet == 0x80: + if ctx.get("bered", False): + ber_encoded = True + else: + raise DecodeError("non normalized arc encoding") arc = (arc << 7) | (octet & 0x7F) if octet & 0x80 == 0: arcs.append(arc) @@ -3229,6 +3289,8 @@ class ObjectIdentifier(Obj): optional=self.optional, _decoded=(offset, llen, l), ) + if ber_encoded: + obj.ber_encoded = True return obj, tail def __repr__(self): @@ -3236,6 +3298,7 @@ class ObjectIdentifier(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -3253,6 +3316,7 @@ class ObjectIdentifier(Obj): expl_llen=self.expl_llen if self.expled else None, expl_vlen=self.expl_vlen if self.expled else None, expl_lenindef=self.expl_lenindef, + ber_encoded=self.ber_encoded, bered=self.bered, ) for pp in self.pps_lenindef(decode_path): @@ -3296,7 +3360,10 @@ class Enumerated(Integer): if isinstance(value, self.__class__): value = value._value elif isinstance(value, integer_types): - if value not in list(self.specs.values()): + for _value in itervalues(self.specs): + if _value == value: + break + else: raise DecodeError( "unknown integer value: %s" % value, klass=self.__class__, @@ -3321,6 +3388,9 @@ class Enumerated(Integer): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __call__( @@ -3462,6 +3532,7 @@ class CommonString(OctetString): if self.ready: value = hexenc(bytes(self)) if no_unicode else self.__unicode__() yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -3498,7 +3569,7 @@ class AllowableCharsMixin(object): def allowable_chars(self): if PY2: return self._allowable_chars - return set(six_unichr(c) for c in self._allowable_chars) + return frozenset(six_unichr(c) for c in self._allowable_chars) class NumericString(AllowableCharsMixin, CommonString): @@ -3514,11 +3585,11 @@ class NumericString(AllowableCharsMixin, CommonString): tag_default = tag_encode(18) encoding = "ascii" asn1_type_name = "NumericString" - _allowable_chars = set(digits.encode("ascii") + b" ") + _allowable_chars = frozenset(digits.encode("ascii") + b" ") def _value_sanitize(self, value): value = super(NumericString, self)._value_sanitize(value) - if not set(value) <= self._allowable_chars: + if not frozenset(value) <= self._allowable_chars: raise DecodeError("non-numeric value") return value @@ -3535,13 +3606,13 @@ class PrintableString(AllowableCharsMixin, CommonString): tag_default = tag_encode(19) encoding = "ascii" asn1_type_name = "PrintableString" - _allowable_chars = set( + _allowable_chars = frozenset( (ascii_letters + digits + " '()+,-./:=?").encode("ascii") ) def _value_sanitize(self, value): value = super(PrintableString, self)._value_sanitize(value) - if not set(value) <= self._allowable_chars: + if not frozenset(value) <= self._allowable_chars: raise DecodeError("non-printable value") return value @@ -3637,10 +3708,6 @@ class UTCTime(CommonString): self._value = default def _value_sanitize(self, value): - if isinstance(value, self.__class__): - return value._value - if isinstance(value, datetime): - return value.strftime(self.fmt).encode("ascii") if isinstance(value, binary_type): try: value_decoded = value.decode("ascii") @@ -3654,6 +3721,10 @@ class UTCTime(CommonString): return value else: raise DecodeError("invalid UTCTime length") + if isinstance(value, self.__class__): + return value._value + if isinstance(value, datetime): + return value.strftime(self.fmt).encode("ascii") raise InvalidValueType((self.__class__, datetime)) def __eq__(self, their): @@ -3694,6 +3765,7 @@ class UTCTime(CommonString): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -3738,12 +3810,6 @@ class GeneralizedTime(UTCTime): fmt_ms = "%Y%m%d%H%M%S.%fZ" def _value_sanitize(self, value): - if isinstance(value, self.__class__): - return value._value - if isinstance(value, datetime): - return value.strftime( - self.fmt_ms if value.microsecond > 0 else self.fmt - ).encode("ascii") if isinstance(value, binary_type): try: value_decoded = value.decode("ascii") @@ -3770,6 +3836,12 @@ class GeneralizedTime(UTCTime): "invalid GeneralizedTime length", klass=self.__class__, ) + if isinstance(value, self.__class__): + return value._value + if isinstance(value, datetime): + return value.strftime( + self.fmt_ms if value.microsecond > 0 else self.fmt + ).encode("ascii") raise InvalidValueType((self.__class__, datetime)) def todatetime(self): @@ -3895,8 +3967,6 @@ class Choice(Obj): self._value = default_obj.copy()._value def _value_sanitize(self, value): - if isinstance(value, self.__class__): - return value._value if isinstance(value, tuple) and len(value) == 2: choice, obj = value spec = self.specs.get(choice) @@ -3905,6 +3975,8 @@ class Choice(Obj): if not isinstance(obj, spec.__class__): raise InvalidValueType((spec,)) return (choice, spec(obj)) + if isinstance(value, self.__class__): + return value._value raise InvalidValueType((self.__class__, tuple)) @property @@ -3926,6 +3998,9 @@ class Choice(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded value = self._value if value is not None: obj._value = (value[0], value[1].copy()) @@ -3997,7 +4072,7 @@ class Choice(Obj): return self._value[1].encode() def _decode(self, tlv, offset, decode_path, ctx, tag_only): - for choice, spec in self.specs.items(): + for choice, spec in iteritems(self.specs): sub_decode_path = decode_path + (choice,) try: spec.decode( @@ -4007,6 +4082,7 @@ class Choice(Obj): decode_path=sub_decode_path, ctx=ctx, tag_only=True, + _ctx_immutable=False, ) except TagMismatch: continue @@ -4025,6 +4101,7 @@ class Choice(Obj): leavemm=True, decode_path=sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) obj = self.__class__( schema=self.specs, @@ -4044,6 +4121,7 @@ class Choice(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -4134,12 +4212,12 @@ class Any(Obj): self.defined = None def _value_sanitize(self, value): + if isinstance(value, binary_type): + return value if isinstance(value, self.__class__): return value._value if isinstance(value, Obj): return value.encode() - if isinstance(value, binary_type): - return value raise InvalidValueType((self.__class__, Obj, binary_type)) @property @@ -4163,6 +4241,9 @@ class Any(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded return obj def __eq__(self, their): @@ -4226,6 +4307,7 @@ class Any(Obj): decode_path=decode_path + (str(chunk_i),), leavemm=True, ctx=ctx, + _ctx_immutable=False, ) vlen += chunk.tlvlen sub_offset += chunk.tlvlen @@ -4270,6 +4352,7 @@ class Any(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -4474,7 +4557,7 @@ class Sequence(Obj): @property def ready(self): - for name, spec in self.specs.items(): + for name, spec in iteritems(self.specs): value = self._value.get(name) if value is None: if spec.optional: @@ -4489,7 +4572,7 @@ class Sequence(Obj): def bered(self): if self.expl_lenindef or self.lenindef or self.ber_encoded: return True - return any(value.bered for value in self._value.values()) + return any(value.bered for value in itervalues(self._value)) def copy(self): obj = self.__class__(schema=self.specs) @@ -4500,7 +4583,10 @@ class Sequence(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen - obj._value = {k: v.copy() for k, v in self._value.items()} + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded + obj._value = {k: v.copy() for k, v in iteritems(self._value)} return obj def __eq__(self, their): @@ -4561,7 +4647,7 @@ class Sequence(Obj): def _encoded_values(self): raws = [] - for name, spec in self.specs.items(): + for name, spec in iteritems(self.specs): value = self._value.get(name) if value is None: if spec.optional: @@ -4627,7 +4713,7 @@ class Sequence(Obj): values = {} ber_encoded = False ctx_allow_default_values = ctx.get("allow_default_values", False) - for name, spec in self.specs.items(): + for name, spec in iteritems(self.specs): if spec.optional and ( (lenindef and v[:EOC_LEN].tobytes() == EOC) or len(v) == 0 @@ -4641,6 +4727,7 @@ class Sequence(Obj): leavemm=True, decode_path=sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) except TagMismatch: if spec.optional: @@ -4665,6 +4752,7 @@ class Sequence(Obj): leavemm=True, decode_path=sub_sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) if len(defined_tail) > 0: raise DecodeError( @@ -4684,6 +4772,7 @@ class Sequence(Obj): leavemm=True, decode_path=sub_decode_path + (DecodePathDefBy(defined_by),), ctx=ctx, + _ctx_immutable=False, ) if len(defined_tail) > 0: raise DecodeError( @@ -4765,6 +4854,7 @@ class Sequence(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -4817,6 +4907,9 @@ class Set(Sequence): v = b"".join(raws) return b"".join((self.tag, len_encode(len(v)), v)) + def _specs_items(self): + return iteritems(self.specs) + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, tlen, lv = tag_strip(tlv) @@ -4871,11 +4964,11 @@ class Set(Sequence): ctx_allow_default_values = ctx.get("allow_default_values", False) ctx_allow_unordered_set = ctx.get("allow_unordered_set", False) value_prev = memoryview(v[:0]) - specs_items = self.specs.items + while len(v) > 0: if lenindef and v[:EOC_LEN].tobytes() == EOC: break - for name, spec in specs_items(): + for name, spec in self._specs_items(): sub_decode_path = decode_path + (name,) try: spec.decode( @@ -4885,6 +4978,7 @@ class Set(Sequence): decode_path=sub_decode_path, ctx=ctx, tag_only=True, + _ctx_immutable=False, ) except TagMismatch: continue @@ -4901,6 +4995,7 @@ class Set(Sequence): leavemm=True, decode_path=sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) value_len = value.fulllen if value_prev.tobytes() > v[:value_len].tobytes(): @@ -5071,6 +5166,9 @@ class SequenceOf(Obj): obj.offset = self.offset obj.llen = self.llen obj.vlen = self.vlen + obj.expl_lenindef = self.expl_lenindef + obj.lenindef = self.lenindef + obj.ber_encoded = self.ber_encoded obj._value = [v.copy() for v in self._value] return obj @@ -5210,6 +5308,7 @@ class SequenceOf(Obj): leavemm=True, decode_path=sub_decode_path, ctx=ctx, + _ctx_immutable=False, ) value_len = value.fulllen if ordering_check: @@ -5267,6 +5366,7 @@ class SequenceOf(Obj): def pps(self, decode_path=()): yield _pp( + obj=self, asn1_type_name=self.asn1_type_name, obj_name=self.__class__.__name__, decode_path=decode_path, @@ -5342,7 +5442,7 @@ def generic_decoder(): # pragma: no cover choice = PrimitiveTypes() choice.specs["SequenceOf"] = SequenceOf(schema=choice) choice.specs["SetOf"] = SetOf(schema=choice) - for i in range(31): + for i in six_xrange(31): choice.specs["SequenceOf%d" % i] = SequenceOf( schema=choice, expl=tag_ctxc(i),