]> Cypherpunks.ru repositories - pyderasn.git/blobdiff - pyderasn.py
Ability to set values during Sequence initialization
[pyderasn.git] / pyderasn.py
index cc44e24c91293ca4ffe85ba7684279e87718925c..68d67d07e91a746886d22bb629d19a8ae6000f43 100755 (executable)
@@ -269,11 +269,11 @@ for AlgorithmIdentifier of X.509's
 ``tbsCertificate.subjectPublicKeyInfo.algorithm.algorithm``::
 
         (
-            (('parameters',), {
+            (("parameters",), {
                 id_ecPublicKey: ECParameters(),
                 id_GostR3410_2001: GostR34102001PublicKeyParameters(),
             }),
-            (('..', 'subjectPublicKey'), {
+            (("..", "subjectPublicKey"), {
                 id_rsaEncryption: RSAPublicKey(),
                 id_GostR3410_2001: OctetString(),
             }),
@@ -289,7 +289,7 @@ Following types can be automatically decoded (DEFINED BY):
 * :py:class:`pyderasn.BitString` (that is multiple of 8 bits)
 * :py:class:`pyderasn.OctetString`
 * :py:class:`pyderasn.SequenceOf`/:py:class:`pyderasn.SetOf`
-  ``Any``/``OctetString``-s
+  ``Any``/``BitString``/``OctetString``-s
 
 When any of those fields is automatically decoded, then ``.defined``
 attribute contains ``(OID, value)`` tuple. ``OID`` tells by which OID it
@@ -329,7 +329,7 @@ of ``PKIResponse``::
         (
             (
                 "content",
-                decode_path_defby(id_signedData),
+                DecodePathDefBy(id_signedData),
                 "encapContentInfo",
                 "eContentType",
             ),
@@ -341,10 +341,10 @@ of ``PKIResponse``::
         (
             (
                 "content",
-                decode_path_defby(id_signedData),
+                DecodePathDefBy(id_signedData),
                 "encapContentInfo",
                 "eContent",
-                decode_path_defby(id_cct_PKIResponse),
+                DecodePathDefBy(id_cct_PKIResponse),
                 "controlSequence",
                 any,
                 "attrType",
@@ -358,7 +358,7 @@ of ``PKIResponse``::
         ),
     ))
 
-Pay attention for :py:func:`pyderasn.decode_path_defby` and ``any``.
+Pay attention for :py:class:`pyderasn.DecodePathDefBy` and ``any``.
 First function is useful for path construction when some automatic
 decoding is already done. ``any`` means literally any value it meet --
 useful for SEQUENCE/SET OF-s.
@@ -472,6 +472,8 @@ from collections import namedtuple
 from collections import OrderedDict
 from datetime import datetime
 from math import ceil
+from os import environ
+from string import digits
 
 from six import add_metaclass
 from six import binary_type
@@ -486,6 +488,13 @@ from six import text_type
 from six.moves import xrange as six_xrange
 
 
+try:
+    from termcolor import colored
+except ImportError:
+    def colored(what, *args):
+        return what
+
+
 __all__ = (
     "Any",
     "BitString",
@@ -493,8 +502,8 @@ __all__ = (
     "Boolean",
     "BoundsError",
     "Choice",
-    "decode_path_defby",
     "DecodeError",
+    "DecodePathDefBy",
     "Enumerated",
     "GeneralizedTime",
     "GeneralString",
@@ -581,7 +590,7 @@ class DecodeError(Exception):
             c for c in (
                 "" if self.klass is None else self.klass.__name__,
                 (
-                    ("(%s)" % ".".join(self.decode_path))
+                    ("(%s)" % ".".join(str(dp) for dp in self.decode_path))
                     if len(self.decode_path) > 0 else ""
                 ),
                 ("(at %d)" % self.offset) if self.offset > 0 else "",
@@ -903,7 +912,7 @@ class Obj(object):
     def _encode(self):  # pragma: no cover
         raise NotImplementedError()
 
-    def _decode(self, tlv, offset, decode_path, ctx):  # pragma: no cover
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):  # pragma: no cover
         raise NotImplementedError()
 
     def encode(self):
@@ -912,7 +921,15 @@ class Obj(object):
             return raw
         return b"".join((self._expl, len_encode(len(raw)), raw))
 
-    def decode(self, data, offset=0, leavemm=False, decode_path=(), ctx=None):
+    def decode(
+            self,
+            data,
+            offset=0,
+            leavemm=False,
+            decode_path=(),
+            ctx=None,
+            tag_only=False,
+    ):
         """Decode the data
 
         :param data: either binary or memoryview
@@ -920,18 +937,25 @@ class Obj(object):
         :param bool leavemm: do we need to leave memoryview of remaining
                     data as is, or convert it to bytes otherwise
         :param ctx: optional :ref:`context <ctx>` governing decoding process.
+        :param tag_only: decode only the tag, without length and contents
+                         (used only in Choice and Set structures, trying to
+                         determine if tag satisfies the scheme)
         :returns: (Obj, remaining data)
         """
         if ctx is None:
             ctx = {}
         tlv = memoryview(data)
         if self._expl is None:
-            obj, tail = self._decode(
+            result = self._decode(
                 tlv,
                 offset,
                 decode_path=decode_path,
                 ctx=ctx,
+                tag_only=tag_only,
             )
+            if tag_only:
+                return
+            obj, tail = result
         else:
             try:
                 t, tlen, lv = tag_strip(tlv)
@@ -964,12 +988,16 @@ class Obj(object):
                     decode_path=decode_path,
                     offset=offset,
                 )
-            obj, tail = self._decode(
+            result = self._decode(
                 v,
                 offset=offset + tlen + llen,
                 decode_path=decode_path,
                 ctx=ctx,
+                tag_only=tag_only,
             )
+            if tag_only:
+                return
+            obj, tail = result
         return obj, (tail if leavemm else tail.tobytes())
 
     @property
@@ -1001,10 +1029,27 @@ class Obj(object):
         return self.expl_tlen + self.expl_llen + self.expl_vlen
 
 
-def decode_path_defby(defined_by):
+class DecodePathDefBy(object):
     """DEFINED BY representation inside decode path
     """
-    return "DEFINED BY (%s)" % defined_by
+    __slots__ = ("defined_by",)
+
+    def __init__(self, defined_by):
+        self.defined_by = defined_by
+
+    def __ne__(self, their):
+        return not(self == their)
+
+    def __eq__(self, their):
+        if not isinstance(their, self.__class__):
+            return False
+        return self.defined_by == their.defined_by
+
+    def __str__(self):
+        return "DEFINED BY " + str(self.defined_by)
+
+    def __repr__(self):
+        return "<%s: %s>" % (self.__class__.__name__, self.defined_by)
 
 
 ########################################################################
@@ -1072,49 +1117,75 @@ def _pp(
     )
 
 
-def pp_console_row(pp, oids=None, with_offsets=False, with_blob=True):
+def _colorize(what, colour, with_colours, attrs=("bold",)):
+    return colored(what, colour, attrs=attrs) if with_colours else what
+
+
+def pp_console_row(
+        pp,
+        oids=None,
+        with_offsets=False,
+        with_blob=True,
+        with_colours=False,
+):
     cols = []
     if with_offsets:
-        cols.append("%5d%s [%d,%d,%4d]" % (
+        col = "%5d%s" % (
             pp.offset,
             (
                 "  " if pp.expl_offset is None else
                 ("-%d" % (pp.offset - pp.expl_offset))
             ),
-            pp.tlen,
-            pp.llen,
-            pp.vlen,
-        ))
+        )
+        cols.append(_colorize(col, "red", with_colours, ()))
+        col = "[%d,%d,%4d]" % (pp.tlen, pp.llen, pp.vlen)
+        cols.append(_colorize(col, "green", with_colours, ()))
     if len(pp.decode_path) > 0:
         cols.append(" ." * (len(pp.decode_path)))
-        cols.append("%s:" % pp.decode_path[-1])
+        ent = pp.decode_path[-1]
+        if isinstance(ent, DecodePathDefBy):
+            cols.append(_colorize("DEFINED BY", "red", with_colours, ("reverse",)))
+            value = str(ent.defined_by)
+            if (
+                    oids is not None and
+                    ent.defined_by.asn1_type_name ==
+                    ObjectIdentifier.asn1_type_name and
+                    value in oids
+            ):
+                cols.append(_colorize("%s:" % oids[value], "green", with_colours))
+            else:
+                cols.append(_colorize("%s:" % value, "white", with_colours, ("reverse",)))
+        else:
+            cols.append(_colorize("%s:" % ent, "yellow", with_colours, ("reverse",)))
     if pp.expl is not None:
         klass, _, num = pp.expl
-        cols.append("[%s%d] EXPLICIT" % (TagClassReprs[klass], num))
+        col = "[%s%d] EXPLICIT" % (TagClassReprs[klass], num)
+        cols.append(_colorize(col, "blue", with_colours))
     if pp.impl is not None:
         klass, _, num = pp.impl
-        cols.append("[%s%d]" % (TagClassReprs[klass], num))
+        col = "[%s%d]" % (TagClassReprs[klass], num)
+        cols.append(_colorize(col, "blue", with_colours))
     if pp.asn1_type_name.replace(" ", "") != pp.obj_name.upper():
-        cols.append(pp.obj_name)
-    cols.append(pp.asn1_type_name)
+        cols.append(_colorize(pp.obj_name, "magenta", with_colours))
+    cols.append(_colorize(pp.asn1_type_name, "cyan", with_colours))
     if pp.value is not None:
         value = pp.value
+        cols.append(_colorize(value, "white", with_colours, ("reverse",)))
         if (
                 oids is not None and
                 pp.asn1_type_name == ObjectIdentifier.asn1_type_name and
                 value in oids
         ):
-            value = "%s (%s)" % (oids[value], pp.value)
-        cols.append(value)
+            cols.append(_colorize("(%s)" % oids[value], "green", with_colours))
     if with_blob:
         if isinstance(pp.blob, binary_type):
             cols.append(hexenc(pp.blob))
         elif isinstance(pp.blob, tuple):
             cols.append(", ".join(pp.blob))
     if pp.optional:
-        cols.append("OPTIONAL")
+        cols.append(_colorize("OPTIONAL", "red", with_colours))
     if pp.default:
-        cols.append("DEFAULT")
+        cols.append(_colorize("DEFAULT", "red", with_colours))
     return " ".join(cols)
 
 
@@ -1133,7 +1204,7 @@ def pp_console_blob(pp):
         yield " ".join(cols + [", ".join(pp.blob)])
 
 
-def pprint(obj, oids=None, big_blobs=False):
+def pprint(obj, oids=None, big_blobs=False, with_colours=False):
     """Pretty print object
 
     :param Obj obj: object you want to pretty print
@@ -1142,6 +1213,8 @@ def pprint(obj, oids=None, big_blobs=False):
     :param big_blobs: if large binary objects are met (like OctetString
                       values), do we need to print them too, on separate
                       lines
+    :param with_colours: colourize output, if ``termcolor`` library
+                         is available
     """
     def _pprint_pps(pps):
         for pp in pps:
@@ -1152,11 +1225,18 @@ def pprint(obj, oids=None, big_blobs=False):
                         oids=oids,
                         with_offsets=True,
                         with_blob=False,
+                        with_colours=with_colours,
                     )
                     for row in pp_console_blob(pp):
                         yield row
                 else:
-                    yield pp_console_row(pp, oids=oids, with_offsets=True)
+                    yield pp_console_row(
+                        pp,
+                        oids=oids,
+                        with_offsets=True,
+                        with_blob=True,
+                        with_colours=with_colours,
+                    )
             else:
                 for row in _pprint_pps(pp):
                     yield row
@@ -1276,7 +1356,7 @@ class Boolean(Obj):
             (b"\xFF" if self._value else b"\x00"),
         ))
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, _, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -1292,6 +1372,8 @@ class Boolean(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, _, v = len_decode(lv)
         except DecodeError as err:
@@ -1569,7 +1651,7 @@ class Integer(Obj):
                     break
         return b"".join((self.tag, len_encode(len(octets)), octets))
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, _, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -1585,6 +1667,8 @@ class Integer(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, llen, v = len_decode(lv)
         except DecodeError as err:
@@ -1708,12 +1792,12 @@ class BitString(Obj):
 
         class KeyUsage(BitString):
             schema = (
-                ('digitalSignature', 0),
-                ('nonRepudiation', 1),
-                ('keyEncipherment', 2),
+                ("digitalSignature", 0),
+                ("nonRepudiation", 1),
+                ("keyEncipherment", 2),
             )
 
-    >>> b = KeyUsage(('keyEncipherment', 'nonRepudiation'))
+    >>> b = KeyUsage(("keyEncipherment", "nonRepudiation"))
     KeyUsage BIT STRING 3 bits nonRepudiation, keyEncipherment
     >>> b.named
     ['nonRepudiation', 'keyEncipherment']
@@ -1903,7 +1987,7 @@ class BitString(Obj):
             octets,
         ))
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, _, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -1919,6 +2003,8 @@ class BitString(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, llen, v = len_decode(lv)
         except DecodeError as err:
@@ -1957,7 +2043,7 @@ class BitString(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
-        if byte2int(v[-1:]) & ((1 << pad_size) - 1) != 0:
+        if byte2int(v[l - 1:l]) & ((1 << pad_size) - 1) != 0:
             raise DecodeError(
                 "invalid pad",
                 klass=self.__class__,
@@ -2009,7 +2095,7 @@ class BitString(Obj):
         defined_by, defined = self.defined or (None, None)
         if defined_by is not None:
             yield defined.pps(
-                decode_path=decode_path + (decode_path_defby(defined_by),)
+                decode_path=decode_path + (DecodePathDefBy(defined_by),)
             )
 
 
@@ -2155,7 +2241,7 @@ class OctetString(Obj):
             self._value,
         ))
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, _, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -2171,6 +2257,8 @@ class OctetString(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, llen, v = len_decode(lv)
         except DecodeError as err:
@@ -2198,6 +2286,13 @@ class OctetString(Obj):
                 optional=self.optional,
                 _decoded=(offset, llen, l),
             )
+        except DecodeError as err:
+            raise DecodeError(
+                msg=err.msg,
+                klass=self.__class__,
+                decode_path=decode_path,
+                offset=offset,
+            )
         except BoundsError as err:
             raise DecodeError(
                 msg=str(err),
@@ -2233,7 +2328,7 @@ class OctetString(Obj):
         defined_by, defined = self.defined or (None, None)
         if defined_by is not None:
             yield defined.pps(
-                decode_path=decode_path + (decode_path_defby(defined_by),)
+                decode_path=decode_path + (DecodePathDefBy(defined_by),)
             )
 
 
@@ -2304,7 +2399,7 @@ class Null(Obj):
     def _encode(self):
         return self.tag + len_encode(0)
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, _, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -2320,6 +2415,8 @@ class Null(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, _, v = len_decode(lv)
         except DecodeError as err:
@@ -2549,7 +2646,7 @@ class ObjectIdentifier(Obj):
         v = b"".join(octets)
         return b"".join((self.tag, len_encode(len(v)), v))
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, _, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -2565,6 +2662,8 @@ class ObjectIdentifier(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, llen, v = len_decode(lv)
         except DecodeError as err:
@@ -2754,7 +2853,7 @@ class CommonString(OctetString):
 
     >>> PrintableString("привет мир")
     Traceback (most recent call last):
-    UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-5: ordinal not in range(128)
+    pyderasn.DecodeError: 'ascii' codec can't encode characters in position 0-5: ordinal not in range(128)
 
     >>> BMPString("ада", bounds=(2, 2))
     Traceback (most recent call last):
@@ -2810,14 +2909,17 @@ class CommonString(OctetString):
             value_raw = value
         else:
             raise InvalidValueType((self.__class__, text_type, binary_type))
-        value_raw = (
-            value_decoded.encode(self.encoding)
-            if value_raw is None else value_raw
-        )
-        value_decoded = (
-            value_raw.decode(self.encoding)
-            if value_decoded is None else value_decoded
-        )
+        try:
+            value_raw = (
+                value_decoded.encode(self.encoding)
+                if value_raw is None else value_raw
+            )
+            value_decoded = (
+                value_raw.decode(self.encoding)
+                if value_decoded is None else value_decoded
+            )
+        except (UnicodeEncodeError, UnicodeDecodeError) as err:
+            raise DecodeError(str(err))
         if not self._bound_min <= len(value_decoded) <= self._bound_max:
             raise BoundsError(
                 self._bound_min,
@@ -2879,6 +2981,13 @@ class NumericString(CommonString):
     tag_default = tag_encode(18)
     encoding = "ascii"
     asn1_type_name = "NumericString"
+    allowable_chars = set(digits.encode("ascii"))
+
+    def _value_sanitize(self, value):
+        value = super(NumericString, self)._value_sanitize(value)
+        if not set(value) <= self.allowable_chars:
+            raise DecodeError("non-numeric value")
+        return value
 
 
 class PrintableString(CommonString):
@@ -3153,8 +3262,8 @@ class Choice(Obj):
 
         class GeneralName(Choice):
             schema = (
-                ('rfc822Name', IA5String(impl=tag_ctxp(1))),
-                ('dNSName', IA5String(impl=tag_ctxp(2))),
+                ("rfc822Name", IA5String(impl=tag_ctxp(1))),
+                ("dNSName", IA5String(impl=tag_ctxp(2))),
             )
 
     >>> gn = GeneralName()
@@ -3316,32 +3425,45 @@ class Choice(Obj):
         self._assert_ready()
         return self._value[1].encode()
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         for choice, spec in self.specs.items():
+            sub_decode_path = decode_path + (choice,)
             try:
-                value, tail = spec.decode(
+                spec.decode(
                     tlv,
                     offset=offset,
                     leavemm=True,
-                    decode_path=decode_path + (choice,),
+                    decode_path=sub_decode_path,
                     ctx=ctx,
+                    tag_only=True,
                 )
             except TagMismatch:
                 continue
-            obj = self.__class__(
-                schema=self.specs,
-                expl=self._expl,
-                default=self.default,
-                optional=self.optional,
-                _decoded=(offset, 0, value.tlvlen),
+            break
+        else:
+            raise TagMismatch(
+                klass=self.__class__,
+                decode_path=decode_path,
+                offset=offset,
             )
-            obj._value = (choice, value)
-            return obj, tail
-        raise TagMismatch(
-            klass=self.__class__,
-            decode_path=decode_path,
+        if tag_only:
+            return
+        value, tail = spec.decode(
+            tlv,
             offset=offset,
+            leavemm=True,
+            decode_path=sub_decode_path,
+            ctx=ctx,
+        )
+        obj = self.__class__(
+            schema=self.specs,
+            expl=self._expl,
+            default=self.default,
+            optional=self.optional,
+            _decoded=(offset, 0, value.tlvlen),
         )
+        obj._value = (choice, value)
+        return obj, tail
 
     def __repr__(self):
         value = pp_console_row(next(self.pps()))
@@ -3491,7 +3613,7 @@ class Any(Obj):
         self._assert_ready()
         return self._value
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, tlen, lv = tag_strip(tlv)
             l, llen, v = len_decode(lv)
@@ -3545,7 +3667,7 @@ class Any(Obj):
         defined_by, defined = self.defined or (None, None)
         if defined_by is not None:
             yield defined.pps(
-                decode_path=decode_path + (decode_path_defby(defined_by),)
+                decode_path=decode_path + (DecodePathDefBy(defined_by),)
             )
 
 
@@ -3618,7 +3740,7 @@ class Sequence(Obj):
     pyderasn.InvalidValueType: invalid value type, expected: <class 'pyderasn.ObjectIdentifier'>
     >>> ext["extnID"] = ObjectIdentifier("1.2.3")
 
-    You can know if sequence is ready to be encoded:
+    You can determine if sequence is ready to be encoded:
 
     >>> ext.ready
     False
@@ -3642,7 +3764,17 @@ class Sequence(Obj):
     >>> tbs = TBSCertificate()
     >>> tbs["version"] = Version("v2") # no need to explicitly add ``expl``
 
-    You can know if value exists/set in the sequence and take its value:
+    Assign ``None`` to remove value from sequence.
+
+    You can set values in Sequence during its initialization:
+
+    >>> AlgorithmIdentifier((
+        ("algorithm", ObjectIdentifier("1.2.3")),
+        ("parameters", Any(Null()))
+    ))
+    AlgorithmIdentifier SEQUENCE[OBJECT IDENTIFIER 1.2.3, ANY 0500 OPTIONAL]
+
+    You can determine if value exists/set in the sequence and take its value:
 
     >>> "extnID" in ext, "extnValue" in ext, "critical" in ext
     (True, True, False)
@@ -3699,9 +3831,17 @@ class Sequence(Obj):
         )
         self._value = {}
         if value is not None:
-            self._value = self._value_sanitize(value)
+            if issubclass(value.__class__, Sequence):
+                self._value = value._value
+            elif hasattr(value, "__iter__"):
+                for seq_key, seq_value in value:
+                    self[seq_key] = seq_value
+            else:
+                raise InvalidValueType((Sequence,))
         if default is not None:
-            default_value = self._value_sanitize(default)
+            if not issubclass(default.__class__, Sequence):
+                raise InvalidValueType((Sequence,))
+            default_value = default._value
             default_obj = self.__class__(impl=self.tag, expl=self._expl)
             default_obj.specs = self.specs
             default_obj._value = default_value
@@ -3709,11 +3849,6 @@ class Sequence(Obj):
             if value is None:
                 self._value = default_obj.copy()._value
 
-    def _value_sanitize(self, value):
-        if not issubclass(value.__class__, Sequence):
-            raise InvalidValueType((Sequence,))
-        return value._value
-
     @property
     def ready(self):
         for name, spec in self.specs.items():
@@ -3810,7 +3945,7 @@ class Sequence(Obj):
         v = b"".join(self._encoded_values())
         return b"".join((self.tag, len_encode(len(v)), v))
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, tlen, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -3826,6 +3961,8 @@ class Sequence(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, llen, v = len_decode(lv)
         except DecodeError as err:
@@ -3869,11 +4006,14 @@ class Sequence(Obj):
                     for i, _value in enumerate(value):
                         sub_sub_decode_path = sub_decode_path + (
                             str(i),
-                            decode_path_defby(defined_by),
+                            DecodePathDefBy(defined_by),
                         )
                         defined_value, defined_tail = defined_spec.decode(
                             memoryview(bytes(_value)),
-                            sub_offset + value.tlen + value.llen,
+                            sub_offset + (
+                                (value.tlen + value.llen + value.expl_tlen + value.expl_llen)
+                                if value.expled else (value.tlen + value.llen)
+                            ),
                             leavemm=True,
                             decode_path=sub_sub_decode_path,
                             ctx=ctx,
@@ -3889,16 +4029,19 @@ class Sequence(Obj):
                 else:
                     defined_value, defined_tail = defined_spec.decode(
                         memoryview(bytes(value)),
-                        sub_offset + value.tlen + value.llen,
+                        sub_offset + (
+                            (value.tlen + value.llen + value.expl_tlen + value.expl_llen)
+                            if value.expled else (value.tlen + value.llen)
+                        ),
                         leavemm=True,
-                        decode_path=sub_decode_path + (decode_path_defby(defined_by),),
+                        decode_path=sub_decode_path + (DecodePathDefBy(defined_by),),
                         ctx=ctx,
                     )
                     if len(defined_tail) > 0:
                         raise DecodeError(
                             "remaining data",
                             klass=self.__class__,
-                            decode_path=sub_decode_path + (decode_path_defby(defined_by),),
+                            decode_path=sub_decode_path + (DecodePathDefBy(defined_by),),
                             offset=offset,
                         )
                     value.defined = (defined_by, defined_value)
@@ -3998,7 +4141,7 @@ class Set(Sequence):
         v = b"".join(raws)
         return b"".join((self.tag, len_encode(len(v)), v))
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, tlen, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -4014,6 +4157,8 @@ class Set(Sequence):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, llen, v = len_decode(lv)
         except DecodeError as err:
@@ -4035,23 +4180,18 @@ class Set(Sequence):
         specs_items = self.specs.items
         while len(v) > 0:
             for name, spec in specs_items():
+                sub_decode_path = decode_path + (name,)
                 try:
-                    value, v_tail = spec.decode(
+                    spec.decode(
                         v,
                         sub_offset,
                         leavemm=True,
-                        decode_path=decode_path + (name,),
+                        decode_path=sub_decode_path,
                         ctx=ctx,
+                        tag_only=True,
                     )
                 except TagMismatch:
                     continue
-                sub_offset += (
-                    value.expl_tlvlen if value.expled else value.tlvlen
-                )
-                v = v_tail
-                if spec.default is None or value != spec.default:  # pragma: no cover
-                    # SeqMixing.test_encoded_default_accepted covers that place
-                    values[name] = value
                 break
             else:
                 raise TagMismatch(
@@ -4059,6 +4199,20 @@ class Set(Sequence):
                     decode_path=decode_path,
                     offset=offset,
                 )
+            value, v_tail = spec.decode(
+                v,
+                sub_offset,
+                leavemm=True,
+                decode_path=sub_decode_path,
+                ctx=ctx,
+            )
+            sub_offset += (
+                value.expl_tlvlen if value.expled else value.tlvlen
+            )
+            v = v_tail
+            if spec.default is None or value != spec.default:  # pragma: no cover
+                # SeqMixing.test_encoded_default_accepted covers that place
+                values[name] = value
         obj = self.__class__(
             schema=self.specs,
             impl=self.tag,
@@ -4251,7 +4405,7 @@ class SequenceOf(Obj):
         v = b"".join(self._encoded_values())
         return b"".join((self.tag, len_encode(len(v)), v))
 
-    def _decode(self, tlv, offset, decode_path, ctx):
+    def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, tlen, lv = tag_strip(tlv)
         except DecodeError as err:
@@ -4267,6 +4421,8 @@ class SequenceOf(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
+        if tag_only:
+            return
         try:
             l, llen, v = len_decode(lv)
         except DecodeError as err:
@@ -4389,7 +4545,7 @@ def generic_decoder():  # pragma: no cover
         __slots__ = ()
         schema = choice
 
-    def pprint_any(obj, oids=None):
+    def pprint_any(obj, oids=None, with_colours=False):
         def _pprint_pps(pps):
             for pp in pps:
                 if hasattr(pp, "_fields"):
@@ -4403,6 +4559,7 @@ def generic_decoder():  # pragma: no cover
                         oids=oids,
                         with_offsets=True,
                         with_blob=False,
+                        with_colours=with_colours,
                     )
                     for row in pp_console_blob(pp):
                         yield row
@@ -4457,7 +4614,11 @@ def main():  # pragma: no cover
             {"defines_by_path": obj_by_path(args.defines_by_path)}
         ),
     )
-    print(pprinter(obj, oids=oids))
+    print(pprinter(
+        obj,
+        oids=oids,
+        with_colours=True if environ.get("NO_COLOR") is None else False,
+    ))
     if tail != b"":
         print("\nTrailing data: %s" % hexenc(tail))