X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=pyderasn.py;h=68d67d07e91a746886d22bb629d19a8ae6000f43;hb=8604ed7e5f423239c839e4bfacd3a585daf0b320;hp=3afaf60fde261bd6a985d06acd63ec4ca0e55fbd;hpb=88244e6e055038f5d09f9cff63dd9507c837f123;p=pyderasn.git diff --git a/pyderasn.py b/pyderasn.py index 3afaf60..68d67d0 100755 --- a/pyderasn.py +++ b/pyderasn.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding: utf-8 # PyDERASN -- Python ASN.1 DER codec with abstract structures -# Copyright (C) 2017 Sergey Matveev +# Copyright (C) 2017-2018 Sergey Matveev # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as @@ -68,7 +68,7 @@ ____ Most types in ASN.1 has specific tag for them. ``Obj.tag_default`` is the default tag used during coding process. You can override it with either ``IMPLICIT`` (using ``impl`` keyword argument), or -``EXPLICIT`` one (using ``expl`` keyword argument). Both arguments takes +``EXPLICIT`` one (using ``expl`` keyword argument). Both arguments take raw binary string, containing that tag. You can **not** set implicit and explicit tags simultaneously. @@ -88,10 +88,10 @@ number. Pay attention that explicit tags always have *constructed* tag Implicit tag is not explicitly shown. -Two object of the same type, but with different implicit/explicit tags +Two objects of the same type, but with different implicit/explicit tags are **not** equal. -You can get objects effective tag (either default or implicited) through +You can get object's effective tag (either default or implicited) through ``tag`` property. You can decode it using :py:func:`pyderasn.tag_decode` function:: @@ -159,12 +159,12 @@ raised. Common methods ______________ -All objects have ``ready`` boolean property, that tells if it is ready -to be encoded. If that kind of action is performed on unready object, -then :py:exc:`pyderasn.ObjNotReady` exception will be raised. +All objects have ``ready`` boolean property, that tells if object is +ready to be encoded. If that kind of action is performed on unready +object, then :py:exc:`pyderasn.ObjNotReady` exception will be raised. -All objects have ``copy()`` method, returning its copy, that can be safely -mutated. +All objects have ``copy()`` method, that returns their copy, that can be +safely mutated. .. _decoding: @@ -195,6 +195,20 @@ lesser than ``offset``), ``expl_tlen``, ``expl_llen``, ``expl_vlen`` When error occurs, then :py:exc:`pyderasn.DecodeError` is raised. +.. _ctx: + +Context +_______ + +You can specify so called context keyword argument during ``decode()`` +invocation. It is dictionary containing various options governing +decoding process. + +Currently available context options: + +* :ref:`defines_by_path ` +* :ref:`strict_default_existence ` + .. _pprinting: Pretty printing @@ -230,15 +244,15 @@ _____________ :py:class:`pyderasn.ObjectIdentifier` field inside :py:class:`pyderasn.Sequence` can hold mapping between OIDs and -necessary for decoding structrures. For example, CMS (:rfc:`5652`) +necessary for decoding structures. For example, CMS (:rfc:`5652`) container:: class ContentInfo(Sequence): schema = ( - ("contentType", ContentType(defines=("content", { + ("contentType", ContentType(defines=((("content",), { id_digestedData: DigestedData(), id_signedData: SignedData(), - }))), + }),))), ("content", Any(expl=tag_ctxc(0))), ) @@ -248,30 +262,52 @@ decoded with ``SignedData`` specification, if ``contentType`` equals to ``contentType`` contains unknown OID, then no automatic decoding is done. +You can specify multiple fields, that will be autodecoded -- that is why +``defines`` kwarg is a sequence. You can specify defined field +relatively or absolutely to current decode path. For example ``defines`` +for AlgorithmIdentifier of X.509's +``tbsCertificate.subjectPublicKeyInfo.algorithm.algorithm``:: + + ( + (("parameters",), { + id_ecPublicKey: ECParameters(), + id_GostR3410_2001: GostR34102001PublicKeyParameters(), + }), + (("..", "subjectPublicKey"), { + id_rsaEncryption: RSAPublicKey(), + id_GostR3410_2001: OctetString(), + }), + ), + +tells that if certificate's SPKI algorithm is GOST R 34.10-2001, then +autodecode its parameters inside SPKI's algorithm and its public key +itself. + Following types can be automatically decoded (DEFINED BY): * :py:class:`pyderasn.Any` +* :py:class:`pyderasn.BitString` (that is multiple of 8 bits) * :py:class:`pyderasn.OctetString` * :py:class:`pyderasn.SequenceOf`/:py:class:`pyderasn.SetOf` - ``Any``/``OctetString``-s + ``Any``/``BitString``/``OctetString``-s When any of those fields is automatically decoded, then ``.defined`` -attribute contains ``(OID, value)`` tuple. OID tell by which OID it was -defined, ``value`` contains corresponding decoded value. For example +attribute contains ``(OID, value)`` tuple. ``OID`` tells by which OID it +was defined, ``value`` contains corresponding decoded value. For example above, ``content_info["content"].defined == (id_signedData, signed_data)``. -.. _defines_by_path_kwarg: +.. _defines_by_path_ctx: -defines_by_path kwarg -_____________________ +defines_by_path context option +______________________________ Sometimes you either can not or do not want to explicitly set *defines* in the scheme. You can dynamically apply those definitions when calling ``.decode()`` method. -Decode method takes optional ``defines_by_path`` keyword argument that -must be sequence of following tuples:: +Specify ``defines_by_path`` key in the :ref:`decode context `. Its +value must be sequence of following tuples:: (decode_path, defines) @@ -288,44 +324,44 @@ of ``PKIResponse``:: content_info, tail = ContentInfo().decode(data, defines_by_path=( ( ("contentType",), - ("content", {id_signedData: SignedData()}), + ((("content",), {id_signedData: SignedData()}),), ), ( ( "content", - decode_path_defby(id_signedData), + DecodePathDefBy(id_signedData), "encapContentInfo", "eContentType", ), - ("eContent", { + ((("eContent",), { id_cct_PKIData: PKIData(), id_cct_PKIResponse: PKIResponse(), - }), + })), ), ( ( "content", - decode_path_defby(id_signedData), + DecodePathDefBy(id_signedData), "encapContentInfo", "eContent", - decode_path_defby(id_cct_PKIResponse), + DecodePathDefBy(id_cct_PKIResponse), "controlSequence", any, "attrType", ), - ("attrValues", { + ((("attrValues",), { id_cmc_recipientNonce: RecipientNonce(), id_cmc_senderNonce: SenderNonce(), id_cmc_statusInfoV2: CMCStatusInfoV2(), id_cmc_transactionId: TransactionId(), - }), + })), ), )) -Pay attention for :py:func:`pyderasn.decode_path_defby` and ``any``. +Pay attention for :py:class:`pyderasn.DecodePathDefBy` and ``any``. First function is useful for path construction when some automatic -decoding is already done. ``any`` is used for human readability and -means literally any value it meet -- useful for sequence and set of-s. +decoding is already done. ``any`` means literally any value it meet -- +useful for SEQUENCE/SET OF-s. Primitive types --------------- @@ -420,6 +456,7 @@ _____ Various ------- +.. autofunction:: pyderasn.abs_decode_path .. autofunction:: pyderasn.hexenc .. autofunction:: pyderasn.hexdec .. autofunction:: pyderasn.tag_encode @@ -435,6 +472,8 @@ from collections import namedtuple from collections import OrderedDict from datetime import datetime from math import ceil +from os import environ +from string import digits from six import add_metaclass from six import binary_type @@ -449,6 +488,13 @@ from six import text_type from six.moves import xrange as six_xrange +try: + from termcolor import colored +except ImportError: + def colored(what, *args): + return what + + __all__ = ( "Any", "BitString", @@ -456,8 +502,8 @@ __all__ = ( "Boolean", "BoundsError", "Choice", - "decode_path_defby", "DecodeError", + "DecodePathDefBy", "Enumerated", "GeneralizedTime", "GeneralString", @@ -544,7 +590,7 @@ class DecodeError(Exception): c for c in ( "" if self.klass is None else self.klass.__name__, ( - ("(%s)" % ".".join(self.decode_path)) + ("(%s)" % ".".join(str(dp) for dp in self.decode_path)) if len(self.decode_path) > 0 else "" ), ("(at %d)" % self.offset) if self.offset > 0 else "", @@ -776,9 +822,9 @@ def len_decode(data): ######################################################################## class AutoAddSlots(type): - def __new__(cls, name, bases, _dict): + def __new__(mcs, name, bases, _dict): _dict["__slots__"] = _dict.get("__slots__", ()) - return type.__new__(cls, name, bases, _dict) + return type.__new__(mcs, name, bases, _dict) @add_metaclass(AutoAddSlots) @@ -866,7 +912,7 @@ class Obj(object): def _encode(self): # pragma: no cover raise NotImplementedError() - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): # pragma: no cover + def _decode(self, tlv, offset, decode_path, ctx, tag_only): # pragma: no cover raise NotImplementedError() def encode(self): @@ -875,24 +921,41 @@ class Obj(object): return raw return b"".join((self._expl, len_encode(len(raw)), raw)) - def decode(self, data, offset=0, leavemm=False, decode_path=(), defines_by_path=None): + def decode( + self, + data, + offset=0, + leavemm=False, + decode_path=(), + ctx=None, + tag_only=False, + ): """Decode the data :param data: either binary or memoryview :param int offset: initial data's offset :param bool leavemm: do we need to leave memoryview of remaining data as is, or convert it to bytes otherwise - :param defines_by_path: :ref:`Read about DEFINED BY ` + :param ctx: optional :ref:`context ` governing decoding process. + :param tag_only: decode only the tag, without length and contents + (used only in Choice and Set structures, trying to + determine if tag satisfies the scheme) :returns: (Obj, remaining data) """ + if ctx is None: + ctx = {} tlv = memoryview(data) if self._expl is None: - obj, tail = self._decode( + result = self._decode( tlv, offset, decode_path=decode_path, - defines_by_path=defines_by_path, + ctx=ctx, + tag_only=tag_only, ) + if tag_only: + return + obj, tail = result else: try: t, tlen, lv = tag_strip(tlv) @@ -925,12 +988,16 @@ class Obj(object): decode_path=decode_path, offset=offset, ) - obj, tail = self._decode( + result = self._decode( v, offset=offset + tlen + llen, decode_path=decode_path, - defines_by_path=defines_by_path, + ctx=ctx, + tag_only=tag_only, ) + if tag_only: + return + obj, tail = result return obj, (tail if leavemm else tail.tobytes()) @property @@ -962,10 +1029,27 @@ class Obj(object): return self.expl_tlen + self.expl_llen + self.expl_vlen -def decode_path_defby(defined_by): +class DecodePathDefBy(object): """DEFINED BY representation inside decode path """ - return "DEFINED BY (%s)" % defined_by + __slots__ = ("defined_by",) + + def __init__(self, defined_by): + self.defined_by = defined_by + + def __ne__(self, their): + return not(self == their) + + def __eq__(self, their): + if not isinstance(their, self.__class__): + return False + return self.defined_by == their.defined_by + + def __str__(self): + return "DEFINED BY " + str(self.defined_by) + + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self.defined_by) ######################################################################## @@ -1033,49 +1117,75 @@ def _pp( ) -def pp_console_row(pp, oids=None, with_offsets=False, with_blob=True): +def _colorize(what, colour, with_colours, attrs=("bold",)): + return colored(what, colour, attrs=attrs) if with_colours else what + + +def pp_console_row( + pp, + oids=None, + with_offsets=False, + with_blob=True, + with_colours=False, +): cols = [] if with_offsets: - cols.append("%5d%s [%d,%d,%4d]" % ( + col = "%5d%s" % ( pp.offset, ( " " if pp.expl_offset is None else ("-%d" % (pp.offset - pp.expl_offset)) ), - pp.tlen, - pp.llen, - pp.vlen, - )) + ) + cols.append(_colorize(col, "red", with_colours, ())) + col = "[%d,%d,%4d]" % (pp.tlen, pp.llen, pp.vlen) + cols.append(_colorize(col, "green", with_colours, ())) if len(pp.decode_path) > 0: cols.append(" ." * (len(pp.decode_path))) - cols.append("%s:" % pp.decode_path[-1]) + ent = pp.decode_path[-1] + if isinstance(ent, DecodePathDefBy): + cols.append(_colorize("DEFINED BY", "red", with_colours, ("reverse",))) + value = str(ent.defined_by) + if ( + oids is not None and + ent.defined_by.asn1_type_name == + ObjectIdentifier.asn1_type_name and + value in oids + ): + cols.append(_colorize("%s:" % oids[value], "green", with_colours)) + else: + cols.append(_colorize("%s:" % value, "white", with_colours, ("reverse",))) + else: + cols.append(_colorize("%s:" % ent, "yellow", with_colours, ("reverse",))) if pp.expl is not None: klass, _, num = pp.expl - cols.append("[%s%d] EXPLICIT" % (TagClassReprs[klass], num)) + col = "[%s%d] EXPLICIT" % (TagClassReprs[klass], num) + cols.append(_colorize(col, "blue", with_colours)) if pp.impl is not None: klass, _, num = pp.impl - cols.append("[%s%d]" % (TagClassReprs[klass], num)) + col = "[%s%d]" % (TagClassReprs[klass], num) + cols.append(_colorize(col, "blue", with_colours)) if pp.asn1_type_name.replace(" ", "") != pp.obj_name.upper(): - cols.append(pp.obj_name) - cols.append(pp.asn1_type_name) + cols.append(_colorize(pp.obj_name, "magenta", with_colours)) + cols.append(_colorize(pp.asn1_type_name, "cyan", with_colours)) if pp.value is not None: value = pp.value + cols.append(_colorize(value, "white", with_colours, ("reverse",))) if ( oids is not None and pp.asn1_type_name == ObjectIdentifier.asn1_type_name and value in oids ): - value = "%s (%s)" % (oids[value], pp.value) - cols.append(value) + cols.append(_colorize("(%s)" % oids[value], "green", with_colours)) if with_blob: if isinstance(pp.blob, binary_type): cols.append(hexenc(pp.blob)) elif isinstance(pp.blob, tuple): cols.append(", ".join(pp.blob)) if pp.optional: - cols.append("OPTIONAL") + cols.append(_colorize("OPTIONAL", "red", with_colours)) if pp.default: - cols.append("DEFAULT") + cols.append(_colorize("DEFAULT", "red", with_colours)) return " ".join(cols) @@ -1094,7 +1204,7 @@ def pp_console_blob(pp): yield " ".join(cols + [", ".join(pp.blob)]) -def pprint(obj, oids=None, big_blobs=False): +def pprint(obj, oids=None, big_blobs=False, with_colours=False): """Pretty print object :param Obj obj: object you want to pretty print @@ -1103,6 +1213,8 @@ def pprint(obj, oids=None, big_blobs=False): :param big_blobs: if large binary objects are met (like OctetString values), do we need to print them too, on separate lines + :param with_colours: colourize output, if ``termcolor`` library + is available """ def _pprint_pps(pps): for pp in pps: @@ -1113,11 +1225,18 @@ def pprint(obj, oids=None, big_blobs=False): oids=oids, with_offsets=True, with_blob=False, + with_colours=with_colours, ) for row in pp_console_blob(pp): yield row else: - yield pp_console_row(pp, oids=oids, with_offsets=True) + yield pp_console_row( + pp, + oids=oids, + with_offsets=True, + with_blob=True, + with_colours=with_colours, + ) else: for row in _pprint_pps(pp): yield row @@ -1237,7 +1356,7 @@ class Boolean(Obj): (b"\xFF" if self._value else b"\x00"), )) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, _, lv = tag_strip(tlv) except DecodeError as err: @@ -1253,6 +1372,8 @@ class Boolean(Obj): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, _, v = len_decode(lv) except DecodeError as err: @@ -1530,7 +1651,7 @@ class Integer(Obj): break return b"".join((self.tag, len_encode(len(octets)), octets)) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, _, lv = tag_strip(tlv) except DecodeError as err: @@ -1546,6 +1667,8 @@ class Integer(Obj): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -1669,19 +1792,19 @@ class BitString(Obj): class KeyUsage(BitString): schema = ( - ('digitalSignature', 0), - ('nonRepudiation', 1), - ('keyEncipherment', 2), + ("digitalSignature", 0), + ("nonRepudiation", 1), + ("keyEncipherment", 2), ) - >>> b = KeyUsage(('keyEncipherment', 'nonRepudiation')) + >>> b = KeyUsage(("keyEncipherment", "nonRepudiation")) KeyUsage BIT STRING 3 bits nonRepudiation, keyEncipherment >>> b.named ['nonRepudiation', 'keyEncipherment'] >>> b.specs {'nonRepudiation': 1, 'digitalSignature': 0, 'keyEncipherment': 2} """ - __slots__ = ("specs",) + __slots__ = ("specs", "defined") tag_default = tag_encode(3) asn1_type_name = "BIT STRING" @@ -1718,6 +1841,7 @@ class BitString(Obj): ) if value is None: self._value = default + self.defined = None def _bits2octets(self, bits): if len(self.specs) > 0: @@ -1863,7 +1987,7 @@ class BitString(Obj): octets, )) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, _, lv = tag_strip(tlv) except DecodeError as err: @@ -1879,6 +2003,8 @@ class BitString(Obj): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -1917,7 +2043,7 @@ class BitString(Obj): decode_path=decode_path, offset=offset, ) - if byte2int(v[-1:]) & ((1 << pad_size) - 1) != 0: + if byte2int(v[l - 1:l]) & ((1 << pad_size) - 1) != 0: raise DecodeError( "invalid pad", klass=self.__class__, @@ -1966,6 +2092,11 @@ class BitString(Obj): expl_llen=self.expl_llen if self.expled else None, expl_vlen=self.expl_vlen if self.expled else None, ) + defined_by, defined = self.defined or (None, None) + if defined_by is not None: + yield defined.pps( + decode_path=decode_path + (DecodePathDefBy(defined_by),) + ) class OctetString(Obj): @@ -2110,7 +2241,7 @@ class OctetString(Obj): self._value, )) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, _, lv = tag_strip(tlv) except DecodeError as err: @@ -2126,6 +2257,8 @@ class OctetString(Obj): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -2153,6 +2286,13 @@ class OctetString(Obj): optional=self.optional, _decoded=(offset, llen, l), ) + except DecodeError as err: + raise DecodeError( + msg=err.msg, + klass=self.__class__, + decode_path=decode_path, + offset=offset, + ) except BoundsError as err: raise DecodeError( msg=str(err), @@ -2188,7 +2328,7 @@ class OctetString(Obj): defined_by, defined = self.defined or (None, None) if defined_by is not None: yield defined.pps( - decode_path=decode_path + (decode_path_defby(defined_by),) + decode_path=decode_path + (DecodePathDefBy(defined_by),) ) @@ -2259,7 +2399,7 @@ class Null(Obj): def _encode(self): return self.tag + len_encode(0) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, _, lv = tag_strip(tlv) except DecodeError as err: @@ -2275,6 +2415,8 @@ class Null(Obj): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, _, v = len_decode(lv) except DecodeError as err: @@ -2346,7 +2488,7 @@ class ObjectIdentifier(Obj): def __init__( self, value=None, - defines=None, + defines=(), impl=None, expl=None, default=None, @@ -2357,12 +2499,14 @@ class ObjectIdentifier(Obj): :param value: set the value. Either tuples of integers, string of "."-concatenated integers, or :py:class:`pyderasn.ObjectIdentifier` object - :param defines: tuple of two elements. First one is a name of - field inside :py:class:`pyderasn.Sequence`, - defining with that OID. Second element is a - ``{OID: pyderasn.Obj()}`` dictionary, mapping - between current OID value and structure applied - to defined field. + :param defines: sequence of tuples. Each tuple has two elements. + First one is relative to current one decode + path, aiming to the field defined by that OID. + Read about relative path in + :py:func:`pyderasn.abs_decode_path`. Second + tuple element is ``{OID: pyderasn.Obj()}`` + dictionary, mapping between current OID value + and structure applied to defined field. :ref:`Read about DEFINED BY ` :param bytes impl: override default tag with ``IMPLICIT`` one :param bytes expl: override default tag with ``EXPLICIT`` one @@ -2502,7 +2646,7 @@ class ObjectIdentifier(Obj): v = b"".join(octets) return b"".join((self.tag, len_encode(len(v)), v)) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, _, lv = tag_strip(tlv) except DecodeError as err: @@ -2518,6 +2662,8 @@ class ObjectIdentifier(Obj): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -2707,7 +2853,7 @@ class CommonString(OctetString): >>> PrintableString("привет мир") Traceback (most recent call last): - UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-5: ordinal not in range(128) + pyderasn.DecodeError: 'ascii' codec can't encode characters in position 0-5: ordinal not in range(128) >>> BMPString("ада", bounds=(2, 2)) Traceback (most recent call last): @@ -2763,14 +2909,17 @@ class CommonString(OctetString): value_raw = value else: raise InvalidValueType((self.__class__, text_type, binary_type)) - value_raw = ( - value_decoded.encode(self.encoding) - if value_raw is None else value_raw - ) - value_decoded = ( - value_raw.decode(self.encoding) - if value_decoded is None else value_decoded - ) + try: + value_raw = ( + value_decoded.encode(self.encoding) + if value_raw is None else value_raw + ) + value_decoded = ( + value_raw.decode(self.encoding) + if value_decoded is None else value_decoded + ) + except (UnicodeEncodeError, UnicodeDecodeError) as err: + raise DecodeError(str(err)) if not self._bound_min <= len(value_decoded) <= self._bound_max: raise BoundsError( self._bound_min, @@ -2832,6 +2981,13 @@ class NumericString(CommonString): tag_default = tag_encode(18) encoding = "ascii" asn1_type_name = "NumericString" + allowable_chars = set(digits.encode("ascii")) + + def _value_sanitize(self, value): + value = super(NumericString, self)._value_sanitize(value) + if not set(value) <= self.allowable_chars: + raise DecodeError("non-numeric value") + return value class PrintableString(CommonString): @@ -3106,8 +3262,8 @@ class Choice(Obj): class GeneralName(Choice): schema = ( - ('rfc822Name', IA5String(impl=tag_ctxp(1))), - ('dNSName', IA5String(impl=tag_ctxp(2))), + ("rfc822Name", IA5String(impl=tag_ctxp(1))), + ("dNSName", IA5String(impl=tag_ctxp(2))), ) >>> gn = GeneralName() @@ -3269,32 +3425,45 @@ class Choice(Obj): self._assert_ready() return self._value[1].encode() - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): for choice, spec in self.specs.items(): + sub_decode_path = decode_path + (choice,) try: - value, tail = spec.decode( + spec.decode( tlv, offset=offset, leavemm=True, - decode_path=decode_path + (choice,), - defines_by_path=defines_by_path, + decode_path=sub_decode_path, + ctx=ctx, + tag_only=True, ) except TagMismatch: continue - obj = self.__class__( - schema=self.specs, - expl=self._expl, - default=self.default, - optional=self.optional, - _decoded=(offset, 0, value.tlvlen), + break + else: + raise TagMismatch( + klass=self.__class__, + decode_path=decode_path, + offset=offset, ) - obj._value = (choice, value) - return obj, tail - raise TagMismatch( - klass=self.__class__, - decode_path=decode_path, + if tag_only: + return + value, tail = spec.decode( + tlv, offset=offset, + leavemm=True, + decode_path=sub_decode_path, + ctx=ctx, ) + obj = self.__class__( + schema=self.specs, + expl=self._expl, + default=self.default, + optional=self.optional, + _decoded=(offset, 0, value.tlvlen), + ) + obj._value = (choice, value) + return obj, tail def __repr__(self): value = pp_console_row(next(self.pps())) @@ -3444,7 +3613,7 @@ class Any(Obj): self._assert_ready() return self._value - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, tlen, lv = tag_strip(tlv) l, llen, v = len_decode(lv) @@ -3498,7 +3667,7 @@ class Any(Obj): defined_by, defined = self.defined or (None, None) if defined_by is not None: yield defined.pps( - decode_path=decode_path + (decode_path_defby(defined_by),) + decode_path=decode_path + (DecodePathDefBy(defined_by),) ) @@ -3519,6 +3688,32 @@ def get_def_by_path(defines_by_path, sub_decode_path): return define +def abs_decode_path(decode_path, rel_path): + """Create an absolute decode path from current and relative ones + + :param decode_path: current decode path, starting point. + Tuple of strings + :param rel_path: relative path to ``decode_path``. Tuple of strings. + If first tuple's element is "/", then treat it as + an absolute path, ignoring ``decode_path`` as + starting point. Also this tuple can contain ".." + elements, stripping the leading element from + ``decode_path`` + + >>> abs_decode_path(("foo", "bar"), ("baz", "whatever")) + ("foo", "bar", "baz", "whatever") + >>> abs_decode_path(("foo", "bar", "baz"), ("..", "..", "whatever")) + ("foo", "whatever") + >>> abs_decode_path(("foo", "bar"), ("/", "baz", "whatever")) + ("baz", "whatever") + """ + if rel_path[0] == "/": + return rel_path[1:] + if rel_path[0] == "..": + return abs_decode_path(decode_path[:-1], rel_path[1:]) + return decode_path + rel_path + + class Sequence(Obj): """``SEQUENCE`` structure type @@ -3545,7 +3740,7 @@ class Sequence(Obj): pyderasn.InvalidValueType: invalid value type, expected: >>> ext["extnID"] = ObjectIdentifier("1.2.3") - You can know if sequence is ready to be encoded: + You can determine if sequence is ready to be encoded: >>> ext.ready False @@ -3569,7 +3764,17 @@ class Sequence(Obj): >>> tbs = TBSCertificate() >>> tbs["version"] = Version("v2") # no need to explicitly add ``expl`` - You can know if value exists/set in the sequence and take its value: + Assign ``None`` to remove value from sequence. + + You can set values in Sequence during its initialization: + + >>> AlgorithmIdentifier(( + ("algorithm", ObjectIdentifier("1.2.3")), + ("parameters", Any(Null())) + )) + AlgorithmIdentifier SEQUENCE[OBJECT IDENTIFIER 1.2.3, ANY 0500 OPTIONAL] + + You can determine if value exists/set in the sequence and take its value: >>> "extnID" in ext, "extnValue" in ext, "critical" in ext (True, True, False) @@ -3588,13 +3793,18 @@ class Sequence(Obj): All defaulted values are always optional. + .. _strict_default_existence_ctx: + .. warning:: When decoded DER contains defaulted value inside, then - technically this is not valid DER encoding. But we allow - and pass it. Of course reencoding of that kind of DER will + technically this is not valid DER encoding. But we allow and pass + it **by default**. Of course reencoding of that kind of DER will result in different binary representation (validly without - defaulted value inside). + defaulted value inside). You can enable strict defaulted values + existence validation by setting ``"strict_default_existence": + True`` :ref:`context ` option -- decoding process will raise + an exception if defaulted value is met. Two sequences are equal if they have equal specification (schema), implicit/explicit tagging and the same values. @@ -3621,9 +3831,17 @@ class Sequence(Obj): ) self._value = {} if value is not None: - self._value = self._value_sanitize(value) + if issubclass(value.__class__, Sequence): + self._value = value._value + elif hasattr(value, "__iter__"): + for seq_key, seq_value in value: + self[seq_key] = seq_value + else: + raise InvalidValueType((Sequence,)) if default is not None: - default_value = self._value_sanitize(default) + if not issubclass(default.__class__, Sequence): + raise InvalidValueType((Sequence,)) + default_value = default._value default_obj = self.__class__(impl=self.tag, expl=self._expl) default_obj.specs = self.specs default_obj._value = default_value @@ -3631,11 +3849,6 @@ class Sequence(Obj): if value is None: self._value = default_obj.copy()._value - def _value_sanitize(self, value): - if not issubclass(value.__class__, Sequence): - raise InvalidValueType((Sequence,)) - return value._value - @property def ready(self): for name, spec in self.specs.items(): @@ -3732,7 +3945,7 @@ class Sequence(Obj): v = b"".join(self._encoded_values()) return b"".join((self.tag, len_encode(len(v)), v)) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, tlen, lv = tag_strip(tlv) except DecodeError as err: @@ -3748,6 +3961,8 @@ class Sequence(Obj): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -3767,7 +3982,6 @@ class Sequence(Obj): v, tail = v[:l], v[l:] sub_offset = offset + tlen + llen values = {} - defines = {} for name, spec in self.specs.items(): if len(v) == 0 and spec.optional: continue @@ -3778,28 +3992,31 @@ class Sequence(Obj): sub_offset, leavemm=True, decode_path=sub_decode_path, - defines_by_path=defines_by_path, + ctx=ctx, ) except TagMismatch: if spec.optional: continue raise - defined = defines.pop(name, None) + defined = get_def_by_path(ctx.get("defines", ()), sub_decode_path) if defined is not None: defined_by, defined_spec = defined if issubclass(value.__class__, SequenceOf): for i, _value in enumerate(value): sub_sub_decode_path = sub_decode_path + ( str(i), - decode_path_defby(defined_by), + DecodePathDefBy(defined_by), ) defined_value, defined_tail = defined_spec.decode( memoryview(bytes(_value)), - sub_offset + value.tlen + value.llen, + sub_offset + ( + (value.tlen + value.llen + value.expl_tlen + value.expl_llen) + if value.expled else (value.tlen + value.llen) + ), leavemm=True, decode_path=sub_sub_decode_path, - defines_by_path=defines_by_path, + ctx=ctx, ) if len(defined_tail) > 0: raise DecodeError( @@ -3812,16 +4029,19 @@ class Sequence(Obj): else: defined_value, defined_tail = defined_spec.decode( memoryview(bytes(value)), - sub_offset + value.tlen + value.llen, + sub_offset + ( + (value.tlen + value.llen + value.expl_tlen + value.expl_llen) + if value.expled else (value.tlen + value.llen) + ), leavemm=True, - decode_path=sub_decode_path + (decode_path_defby(defined_by),), - defines_by_path=defines_by_path, + decode_path=sub_decode_path + (DecodePathDefBy(defined_by),), + ctx=ctx, ) if len(defined_tail) > 0: raise DecodeError( "remaining data", klass=self.__class__, - decode_path=sub_decode_path + (decode_path_defby(defined_by),), + decode_path=sub_decode_path + (DecodePathDefBy(defined_by),), offset=offset, ) value.defined = (defined_by, defined_value) @@ -3829,19 +4049,30 @@ class Sequence(Obj): sub_offset += (value.expl_tlvlen if value.expled else value.tlvlen) v = v_tail if spec.default is not None and value == spec.default: - # Encoded default values are not valid in DER, - # but we allow that anyway - continue + if ctx.get("strict_default_existence", False): + raise DecodeError( + "DEFAULT value met", + klass=self.__class__, + decode_path=sub_decode_path, + offset=sub_offset, + ) + else: + continue values[name] = value - spec_defines = getattr(spec, "defines", None) - if defines_by_path is not None and spec_defines is None: - spec_defines = get_def_by_path(defines_by_path, sub_decode_path) - if spec_defines is not None: - what, schema = spec_defines - defined = schema.get(value, None) - if defined is not None: - defines[what] = (value, defined) + spec_defines = getattr(spec, "defines", ()) + if len(spec_defines) == 0: + defines_by_path = ctx.get("defines_by_path", ()) + if len(defines_by_path) > 0: + spec_defines = get_def_by_path(defines_by_path, sub_decode_path) + if spec_defines is not None and len(spec_defines) > 0: + for rel_path, schema in spec_defines: + defined = schema.get(value, None) + if defined is not None: + ctx.setdefault("defines", []).append(( + abs_decode_path(sub_decode_path[:-1], rel_path), + (value, defined), + )) if len(v) > 0: raise DecodeError( "remaining data", @@ -3910,7 +4141,7 @@ class Set(Sequence): v = b"".join(raws) return b"".join((self.tag, len_encode(len(v)), v)) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, tlen, lv = tag_strip(tlv) except DecodeError as err: @@ -3926,6 +4157,8 @@ class Set(Sequence): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -3947,23 +4180,18 @@ class Set(Sequence): specs_items = self.specs.items while len(v) > 0: for name, spec in specs_items(): + sub_decode_path = decode_path + (name,) try: - value, v_tail = spec.decode( + spec.decode( v, sub_offset, leavemm=True, - decode_path=decode_path + (name,), - defines_by_path=defines_by_path, + decode_path=sub_decode_path, + ctx=ctx, + tag_only=True, ) except TagMismatch: continue - sub_offset += ( - value.expl_tlvlen if value.expled else value.tlvlen - ) - v = v_tail - if spec.default is None or value != spec.default: # pragma: no cover - # SeqMixing.test_encoded_default_accepted covers that place - values[name] = value break else: raise TagMismatch( @@ -3971,6 +4199,20 @@ class Set(Sequence): decode_path=decode_path, offset=offset, ) + value, v_tail = spec.decode( + v, + sub_offset, + leavemm=True, + decode_path=sub_decode_path, + ctx=ctx, + ) + sub_offset += ( + value.expl_tlvlen if value.expled else value.tlvlen + ) + v = v_tail + if spec.default is None or value != spec.default: # pragma: no cover + # SeqMixing.test_encoded_default_accepted covers that place + values[name] = value obj = self.__class__( schema=self.specs, impl=self.tag, @@ -4163,7 +4405,7 @@ class SequenceOf(Obj): v = b"".join(self._encoded_values()) return b"".join((self.tag, len_encode(len(v)), v)) - def _decode(self, tlv, offset=0, decode_path=(), defines_by_path=None): + def _decode(self, tlv, offset, decode_path, ctx, tag_only): try: t, tlen, lv = tag_strip(tlv) except DecodeError as err: @@ -4179,6 +4421,8 @@ class SequenceOf(Obj): decode_path=decode_path, offset=offset, ) + if tag_only: + return try: l, llen, v = len_decode(lv) except DecodeError as err: @@ -4205,7 +4449,7 @@ class SequenceOf(Obj): sub_offset, leavemm=True, decode_path=decode_path + (str(len(_value)),), - defines_by_path=defines_by_path, + ctx=ctx, ) sub_offset += (value.expl_tlvlen if value.expled else value.tlvlen) v = v_tail @@ -4301,7 +4545,7 @@ def generic_decoder(): # pragma: no cover __slots__ = () schema = choice - def pprint_any(obj, oids=None): + def pprint_any(obj, oids=None, with_colours=False): def _pprint_pps(pps): for pp in pps: if hasattr(pp, "_fields"): @@ -4315,6 +4559,7 @@ def generic_decoder(): # pragma: no cover oids=oids, with_offsets=True, with_blob=False, + with_colours=with_colours, ) for row in pp_console_blob(pp): yield row @@ -4364,12 +4609,16 @@ def main(): # pragma: no cover schema, pprinter = generic_decoder() obj, tail = schema().decode( der, - defines_by_path=( - None if args.defines_by_path is None - else obj_by_path(args.defines_by_path) + ctx=( + None if args.defines_by_path is None else + {"defines_by_path": obj_by_path(args.defines_by_path)} ), ) - print(pprinter(obj, oids=oids)) + print(pprinter( + obj, + oids=oids, + with_colours=True if environ.get("NO_COLOR") is None else False, + )) if tail != b"": print("\nTrailing data: %s" % hexenc(tail))