]> Cypherpunks.ru repositories - pyderasn.git/blobdiff - pyderasn.py
Change order of value sanitizers
[pyderasn.git] / pyderasn.py
index 2213e7804b60ae3b403dc09c25f6ad536ef08af0..50faa0be870ced74ab9d33afb4ac0f4008846421 100755 (executable)
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 # PyDERASN -- Python ASN.1 DER/BER codec with abstract structures
-# Copyright (C) 2017-2018 Sergey Matveev <stargrave@stargrave.org>
+# Copyright (C) 2017-2019 Sergey Matveev <stargrave@stargrave.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as
@@ -382,15 +382,22 @@ DER. But you can optionally enable BER decoding with setting ``bered``
 :ref:`context <ctx>` argument to True. Indefinite lengths and
 constructed primitive types should be parsed successfully.
 
-* If object is encoded in BER form (not the DER one), then ``bered``
+* If object is encoded in BER form (not the DER one), then ``ber_encoded``
   attribute is set to True. Only ``BOOLEAN``, ``BIT STRING``, ``OCTET
-  STRING``, ``SEQUENCE``, ``SET``, ``SET OF`` can contain it.
+  STRING``, ``OBJECT IDENTIFIER``, ``SEQUENCE``, ``SET``, ``SET OF``
+  can contain it.
 * If object has an indefinite length encoding, then its ``lenindef``
   attribute is set to True. Only ``BIT STRING``, ``OCTET STRING``,
   ``SEQUENCE``, ``SET``, ``SEQUENCE OF``, ``SET OF``, ``ANY`` can
   contain it.
 * If object has an indefinite length encoded explicit tag, then
   ``expl_lenindef`` is set to True.
+* If object has either any of BER-related encoding (explicit tag
+  indefinite length, object's indefinite length, BER-encoding) or any
+  underlying component has that kind of encoding, then ``bered``
+  attribute is set to True. For example SignedData CMS can have
+  ``ContentInfo:content:signerInfos:*`` ``bered`` value set to True, but
+  ``ContentInfo:content:signerInfos:*:signedAttrs`` won't.
 
 EOC (end-of-contents) token's length is taken in advance in object's
 value length.
@@ -509,6 +516,7 @@ Various
 -------
 
 .. autofunction:: pyderasn.abs_decode_path
+.. autofunction:: pyderasn.colonize_hex
 .. autofunction:: pyderasn.hexenc
 .. autofunction:: pyderasn.hexdec
 .. autofunction:: pyderasn.tag_encode
@@ -533,9 +541,11 @@ from codecs import getdecoder
 from codecs import getencoder
 from collections import namedtuple
 from collections import OrderedDict
+from copy import copy
 from datetime import datetime
 from math import ceil
 from os import environ
+from string import ascii_letters
 from string import digits
 
 from six import add_metaclass
@@ -545,15 +555,18 @@ from six import indexbytes
 from six import int2byte
 from six import integer_types
 from six import iterbytes
+from six import iteritems
+from six import itervalues
 from six import PY2
 from six import string_types
 from six import text_type
+from six import unichr as six_unichr
 from six.moves import xrange as six_xrange
 
 
 try:
     from termcolor import colored
-except ImportError:
+except ImportError:  # pragma: no cover
     def colored(what, *args):
         return what
 
@@ -635,7 +648,11 @@ LENINDEF_PP_CHAR = "I" if PY2 else "∞"
 # Errors
 ########################################################################
 
-class DecodeError(Exception):
+class ASN1Error(ValueError):
+    pass
+
+
+class DecodeError(ASN1Error):
     def __init__(self, msg="", klass=None, decode_path=(), offset=0):
         """
         :param str msg: reason of decode failing
@@ -690,7 +707,7 @@ class InvalidOID(DecodeError):
     pass
 
 
-class ObjUnknown(ValueError):
+class ObjUnknown(ASN1Error):
     def __init__(self, name):
         super(ObjUnknown, self).__init__()
         self.name = name
@@ -702,7 +719,7 @@ class ObjUnknown(ValueError):
         return "%s(%s)" % (self.__class__.__name__, self)
 
 
-class ObjNotReady(ValueError):
+class ObjNotReady(ASN1Error):
     def __init__(self, name):
         super(ObjNotReady, self).__init__()
         self.name = name
@@ -714,7 +731,7 @@ class ObjNotReady(ValueError):
         return "%s(%s)" % (self.__class__.__name__, self)
 
 
-class InvalidValueType(ValueError):
+class InvalidValueType(ASN1Error):
     def __init__(self, expected_types):
         super(InvalidValueType, self).__init__()
         self.expected_types = expected_types
@@ -728,7 +745,7 @@ class InvalidValueType(ValueError):
         return "%s(%s)" % (self.__class__.__name__, self)
 
 
-class BoundsError(ValueError):
+class BoundsError(ASN1Error):
     def __init__(self, bound_min, value, bound_max):
         super(BoundsError, self).__init__()
         self.bound_min = bound_min
@@ -922,7 +939,7 @@ class Obj(object):
         "vlen",
         "expl_lenindef",
         "lenindef",
-        "bered",
+        "ber_encoded",
     )
 
     def __init__(
@@ -944,7 +961,7 @@ class Obj(object):
         self.default = None
         self.expl_lenindef = False
         self.lenindef = False
-        self.bered = False
+        self.ber_encoded = False
 
     @property
     def ready(self):  # pragma: no cover
@@ -956,6 +973,12 @@ class Obj(object):
         if not self.ready:
             raise ObjNotReady(self.__class__.__name__)
 
+    @property
+    def bered(self):
+        """Is either object or any elements inside is BER encoded?
+        """
+        return self.expl_lenindef or self.lenindef or self.ber_encoded
+
     @property
     def decoded(self):
         """Is object decoded?
@@ -1010,6 +1033,7 @@ class Obj(object):
             decode_path=(),
             ctx=None,
             tag_only=False,
+            _ctx_immutable=True,
     ):
         """Decode the data
 
@@ -1017,14 +1041,17 @@ class Obj(object):
         :param int offset: initial data's offset
         :param bool leavemm: do we need to leave memoryview of remaining
                     data as is, or convert it to bytes otherwise
-        :param ctx: optional :ref:`context <ctx>` governing decoding process.
+        :param ctx: optional :ref:`context <ctx>` governing decoding process
         :param tag_only: decode only the tag, without length and contents
                          (used only in Choice and Set structures, trying to
                          determine if tag satisfies the scheme)
+        :param _ctx_immutable: do we need to copy ``ctx`` before using it
         :returns: (Obj, remaining data)
         """
         if ctx is None:
             ctx = {}
+        elif _ctx_immutable:
+            ctx = copy(ctx)
         tlv = memoryview(data)
         if self._expl is None:
             result = self._decode(
@@ -1072,7 +1099,7 @@ class Obj(object):
                     ctx=ctx,
                     tag_only=tag_only,
                 )
-                if tag_only:
+                if tag_only:  # pragma: no cover
                     return
                 obj, tail = result
                 eoc_expected, tail = tail[:EOC_LEN], tail[EOC_LEN:]
@@ -1107,7 +1134,7 @@ class Obj(object):
                     ctx=ctx,
                     tag_only=tag_only,
                 )
-                if tag_only:
+                if tag_only:  # pragma: no cover
                     return
                 obj, tail = result
                 if obj.tlvlen < l and not ctx.get("allow_expl_oob", False):
@@ -1158,7 +1185,10 @@ class Obj(object):
         return self.expl_tlvlen if self.expled else self.tlvlen
 
     def pps_lenindef(self, decode_path):
-        if self.lenindef:
+        if self.lenindef and not (
+            getattr(self, "defined", None) is not None and
+            self.defined[1].lenindef
+        ):
             yield _pp(
                 asn1_type_name="EOC",
                 obj_name="",
@@ -1170,6 +1200,7 @@ class Obj(object):
                 tlen=1,
                 llen=1,
                 vlen=0,
+                ber_encoded=True,
                 bered=True,
             )
         if self.expl_lenindef:
@@ -1181,6 +1212,7 @@ class Obj(object):
                 tlen=1,
                 llen=1,
                 vlen=0,
+                ber_encoded=True,
                 bered=True,
             )
 
@@ -1213,6 +1245,7 @@ class DecodePathDefBy(object):
 ########################################################################
 
 PP = namedtuple("PP", (
+    "obj",
     "asn1_type_name",
     "obj_name",
     "decode_path",
@@ -1232,11 +1265,13 @@ PP = namedtuple("PP", (
     "expl_vlen",
     "expl_lenindef",
     "lenindef",
+    "ber_encoded",
     "bered",
 ))
 
 
 def _pp(
+        obj=None,
         asn1_type_name="unknown",
         obj_name="unknown",
         decode_path=(),
@@ -1256,9 +1291,11 @@ def _pp(
         expl_vlen=None,
         expl_lenindef=False,
         lenindef=False,
+        ber_encoded=False,
         bered=False,
 ):
     return PP(
+        obj,
         asn1_type_name,
         obj_name,
         decode_path,
@@ -1278,6 +1315,7 @@ def _pp(
         expl_vlen,
         expl_lenindef,
         lenindef,
+        ber_encoded,
         bered,
     )
 
@@ -1286,6 +1324,12 @@ def _colourize(what, colour, with_colours, attrs=("bold",)):
     return colored(what, colour, attrs=attrs) if with_colours else what
 
 
+def colonize_hex(hexed):
+    """Separate hexadecimal string with colons
+    """
+    return ":".join(hexed[i:i + 2] for i in six_xrange(0, len(hexed), 2))
+
+
 def pp_console_row(
         pp,
         oids=None,
@@ -1305,7 +1349,9 @@ def pp_console_row(
             ),
             LENINDEF_PP_CHAR if pp.expl_lenindef else " ",
         )
-        cols.append(_colourize(col, "red", with_colours, ()))
+        col = _colourize(col, "red", with_colours, ())
+        col += _colourize("B", "red", with_colours) if pp.bered else " "
+        cols.append(col)
         col = "[%d,%d,%4d]%s" % (
             pp.tlen,
             pp.llen,
@@ -1342,7 +1388,7 @@ def pp_console_row(
         cols.append(_colourize(col, "blue", with_colours))
     if pp.asn1_type_name.replace(" ", "") != pp.obj_name.upper():
         cols.append(_colourize(pp.obj_name, "magenta", with_colours))
-    if pp.bered:
+    if pp.ber_encoded:
         cols.append(_colourize("BER", "red", with_colours))
     cols.append(_colourize(pp.asn1_type_name, "cyan", with_colours))
     if pp.value is not None:
@@ -1354,6 +1400,15 @@ def pp_console_row(
                 value in oids
         ):
             cols.append(_colourize("(%s)" % oids[value], "green", with_colours))
+        if pp.asn1_type_name == Integer.asn1_type_name:
+            hex_repr = hex(int(pp.obj._value))[2:].upper()
+            if len(hex_repr) % 2 != 0:
+                hex_repr = "0" + hex_repr
+            cols.append(_colourize(
+                "(%s)" % colonize_hex(hex_repr),
+                "green",
+                with_colours,
+            ))
     if with_blob:
         if isinstance(pp.blob, binary_type):
             cols.append(hexenc(pp.blob))
@@ -1373,17 +1428,15 @@ def pp_console_row(
 
 
 def pp_console_blob(pp, decode_path_len_decrease=0):
-    cols = [" " * len("XXXXXYYZ [X,X,XXXX]Z")]
+    cols = [" " * len("XXXXXYYZZ [X,X,XXXX]Z")]
     decode_path_len = len(pp.decode_path) - decode_path_len_decrease
     if decode_path_len > 0:
         cols.append(" ." * (decode_path_len + 1))
     if isinstance(pp.blob, binary_type):
         blob = hexenc(pp.blob).upper()
-        for i in range(0, len(blob), 32):
+        for i in six_xrange(0, len(blob), 32):
             chunk = blob[i:i + 32]
-            yield " ".join(cols + [":".join(
-                chunk[j:j + 2] for j in range(0, len(chunk), 2)
-            )])
+            yield " ".join(cols + [colonize_hex(chunk)])
     elif isinstance(pp.blob, tuple):
         yield " ".join(cols + [", ".join(pp.blob)])
 
@@ -1498,10 +1551,10 @@ class Boolean(Obj):
                 self._value = default
 
     def _value_sanitize(self, value):
-        if issubclass(value.__class__, Boolean):
-            return value._value
         if isinstance(value, bool):
             return value
+        if issubclass(value.__class__, Boolean):
+            return value._value
         raise InvalidValueType((self.__class__, bool))
 
     @property
@@ -1518,6 +1571,9 @@ class Boolean(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         return obj
 
     def __nonzero__(self):
@@ -1605,14 +1661,14 @@ class Boolean(Obj):
                 offset=offset,
             )
         first_octet = byte2int(v)
-        bered = False
+        ber_encoded = False
         if first_octet == 0:
             value = False
         elif first_octet == 0xFF:
             value = True
         elif ctx.get("bered", False):
             value = True
-            bered = True
+            ber_encoded = True
         else:
             raise DecodeError(
                 "unacceptable Boolean value",
@@ -1628,7 +1684,7 @@ class Boolean(Obj):
             optional=self.optional,
             _decoded=(offset, 1, 1),
         )
-        obj.bered = bered
+        obj.ber_encoded = ber_encoded
         return obj, v[1:]
 
     def __repr__(self):
@@ -1636,6 +1692,7 @@ class Boolean(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -1653,6 +1710,7 @@ class Boolean(Obj):
             expl_llen=self.expl_llen if self.expled else None,
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
+            ber_encoded=self.ber_encoded,
             bered=self.bered,
         )
         for pp in self.pps_lenindef(decode_path):
@@ -1742,10 +1800,10 @@ class Integer(Obj):
                 self._value = default
 
     def _value_sanitize(self, value):
-        if issubclass(value.__class__, Integer):
-            value = value._value
-        elif isinstance(value, integer_types):
+        if isinstance(value, integer_types):
             pass
+        elif issubclass(value.__class__, Integer):
+            value = value._value
         elif isinstance(value, str):
             value = self.specs.get(value)
             if value is None:
@@ -1772,6 +1830,9 @@ class Integer(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         return obj
 
     def __int__(self):
@@ -1802,7 +1863,7 @@ class Integer(Obj):
 
     @property
     def named(self):
-        for name, value in self.specs.items():
+        for name, value in iteritems(self.specs):
             if value == self._value:
                 return name
 
@@ -1962,6 +2023,7 @@ class Integer(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -1979,11 +2041,15 @@ class Integer(Obj):
             expl_llen=self.expl_llen if self.expled else None,
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
+            bered=self.bered,
         )
         for pp in self.pps_lenindef(decode_path):
             yield pp
 
 
+SET01 = frozenset(("0", "1"))
+
+
 class BitString(Obj):
     """``BIT STRING`` bit string type
 
@@ -2089,8 +2155,6 @@ class BitString(Obj):
         return bit_len, bytes(octets)
 
     def _value_sanitize(self, value):
-        if issubclass(value.__class__, BitString):
-            return value._value
         if isinstance(value, (string_types, binary_type)):
             if (
                     isinstance(value, string_types) and
@@ -2098,7 +2162,7 @@ class BitString(Obj):
             ):
                 if value.endswith("'B"):
                     value = value[1:-2]
-                    if not set(value) <= set(("0", "1")):
+                    if not frozenset(value) <= SET01:
                         raise ValueError("B's coding contains unacceptable chars")
                     return self._bits2octets(value)
                 elif value.endswith("'H"):
@@ -2126,11 +2190,13 @@ class BitString(Obj):
                 bits.append(bit)
             if len(bits) == 0:
                 return self._bits2octets("")
-            bits = set(bits)
+            bits = frozenset(bits)
             return self._bits2octets("".join(
                 ("1" if bit in bits else "0")
                 for bit in six_xrange(max(bits) + 1)
             ))
+        if issubclass(value.__class__, BitString):
+            return value._value
         raise InvalidValueType((self.__class__, binary_type, string_types))
 
     @property
@@ -2150,6 +2216,9 @@ class BitString(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         return obj
 
     def __iter__(self):
@@ -2179,7 +2248,7 @@ class BitString(Obj):
 
     @property
     def named(self):
-        return [name for name, bit in self.specs.items() if self[bit]]
+        return [name for name, bit in iteritems(self.specs) if self[bit]]
 
     def __call__(
             self,
@@ -2293,7 +2362,7 @@ class BitString(Obj):
                 offset=offset,
             )
         if t == self.tag:
-            if tag_only:
+            if tag_only:  # pragma: no cover
                 return
             return self._decode_chunk(lv, offset, decode_path, ctx)
         if t == self.tag_constructed:
@@ -2304,7 +2373,7 @@ class BitString(Obj):
                     decode_path=decode_path,
                     offset=offset,
                 )
-            if tag_only:
+            if tag_only:  # pragma: no cover
                 return
             lenindef = False
             try:
@@ -2358,6 +2427,7 @@ class BitString(Obj):
                         decode_path=sub_decode_path,
                         leavemm=True,
                         ctx=ctx,
+                        _ctx_immutable=False,
                     )
                 except TagMismatch:
                     raise DecodeError(
@@ -2402,7 +2472,7 @@ class BitString(Obj):
                 _decoded=(offset, llen, vlen + (EOC_LEN if lenindef else 0)),
             )
             obj.lenindef = lenindef
-            obj.bered = True
+            obj.ber_encoded = True
             return obj, (v[EOC_LEN:] if lenindef else v)
         raise TagMismatch(
             klass=self.__class__,
@@ -2422,6 +2492,7 @@ class BitString(Obj):
             if len(self.specs) > 0:
                 blob = tuple(self.named)
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -2441,6 +2512,7 @@ class BitString(Obj):
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
             lenindef=self.lenindef,
+            ber_encoded=self.ber_encoded,
             bered=self.bered,
         )
         defined_by, defined = self.defined or (None, None)
@@ -2533,10 +2605,10 @@ class OctetString(Obj):
         )
 
     def _value_sanitize(self, value):
-        if issubclass(value.__class__, OctetString):
-            value = value._value
-        elif isinstance(value, binary_type):
+        if isinstance(value, binary_type):
             pass
+        elif issubclass(value.__class__, OctetString):
+            value = value._value
         else:
             raise InvalidValueType((self.__class__, bytes))
         if not self._bound_min <= len(value) <= self._bound_max:
@@ -2559,6 +2631,9 @@ class OctetString(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         return obj
 
     def __bytes__(self):
@@ -2721,6 +2796,7 @@ class OctetString(Obj):
                         decode_path=sub_decode_path,
                         leavemm=True,
                         ctx=ctx,
+                        _ctx_immutable=False,
                     )
                 except TagMismatch:
                     raise DecodeError(
@@ -2758,7 +2834,7 @@ class OctetString(Obj):
                     offset=offset,
                 )
             obj.lenindef = lenindef
-            obj.bered = True
+            obj.ber_encoded = True
             return obj, (v[EOC_LEN:] if lenindef else v)
         raise TagMismatch(
             klass=self.__class__,
@@ -2771,6 +2847,7 @@ class OctetString(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -2790,6 +2867,7 @@ class OctetString(Obj):
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
             lenindef=self.lenindef,
+            ber_encoded=self.ber_encoded,
             bered=self.bered,
         )
         defined_by, defined = self.defined or (None, None)
@@ -2842,6 +2920,9 @@ class Null(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         return obj
 
     def __eq__(self, their):
@@ -2884,7 +2965,7 @@ class Null(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
-        if tag_only:
+        if tag_only:  # pragma: no cover
             return
         try:
             l, _, v = len_decode(lv)
@@ -2915,6 +2996,7 @@ class Null(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -2930,6 +3012,7 @@ class Null(Obj):
             expl_llen=self.expl_llen if self.expled else None,
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
+            bered=self.bered,
         )
         for pp in self.pps_lenindef(decode_path):
             yield pp
@@ -3050,6 +3133,9 @@ class ObjectIdentifier(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         return obj
 
     def __iter__(self):
@@ -3134,7 +3220,7 @@ class ObjectIdentifier(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
-        if tag_only:
+        if tag_only:  # pragma: no cover
             return
         try:
             l, llen, v = len_decode(lv)
@@ -3161,11 +3247,17 @@ class ObjectIdentifier(Obj):
             )
         v, tail = v[:l], v[l:]
         arcs = []
+        ber_encoded = False
         while len(v) > 0:
             i = 0
             arc = 0
             while True:
                 octet = indexbytes(v, i)
+                if i == 0 and octet == 0x80:
+                    if ctx.get("bered", False):
+                        ber_encoded = True
+                    else:
+                        raise DecodeError("non normalized arc encoding")
                 arc = (arc << 7) | (octet & 0x7F)
                 if octet & 0x80 == 0:
                     arcs.append(arc)
@@ -3197,6 +3289,8 @@ class ObjectIdentifier(Obj):
             optional=self.optional,
             _decoded=(offset, llen, l),
         )
+        if ber_encoded:
+            obj.ber_encoded = True
         return obj, tail
 
     def __repr__(self):
@@ -3204,6 +3298,7 @@ class ObjectIdentifier(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -3221,6 +3316,8 @@ class ObjectIdentifier(Obj):
             expl_llen=self.expl_llen if self.expled else None,
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
+            ber_encoded=self.ber_encoded,
+            bered=self.bered,
         )
         for pp in self.pps_lenindef(decode_path):
             yield pp
@@ -3263,7 +3360,10 @@ class Enumerated(Integer):
         if isinstance(value, self.__class__):
             value = value._value
         elif isinstance(value, integer_types):
-            if value not in list(self.specs.values()):
+            for _value in itervalues(self.specs):
+                if _value == value:
+                    break
+            else:
                 raise DecodeError(
                     "unknown integer value: %s" % value,
                     klass=self.__class__,
@@ -3288,6 +3388,9 @@ class Enumerated(Integer):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         return obj
 
     def __call__(
@@ -3429,6 +3532,7 @@ class CommonString(OctetString):
         if self.ready:
             value = hexenc(bytes(self)) if no_unicode else self.__unicode__()
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -3446,6 +3550,8 @@ class CommonString(OctetString):
             expl_llen=self.expl_llen if self.expled else None,
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
+            ber_encoded=self.ber_encoded,
+            bered=self.bered,
         )
         for pp in self.pps_lenindef(decode_path):
             yield pp
@@ -3458,29 +3564,57 @@ class UTF8String(CommonString):
     asn1_type_name = "UTF8String"
 
 
-class NumericString(CommonString):
+class AllowableCharsMixin(object):
+    @property
+    def allowable_chars(self):
+        if PY2:
+            return self._allowable_chars
+        return frozenset(six_unichr(c) for c in self._allowable_chars)
+
+
+class NumericString(AllowableCharsMixin, CommonString):
     """Numeric string
 
-    Its value is properly sanitized: only ASCII digits can be stored.
+    Its value is properly sanitized: only ASCII digits with spaces can
+    be stored.
+
+    >>> NumericString().allowable_chars
+    set(['3', '4', '7', '5', '1', '0', '8', '9', ' ', '6', '2'])
     """
     __slots__ = ()
     tag_default = tag_encode(18)
     encoding = "ascii"
     asn1_type_name = "NumericString"
-    allowable_chars = set(digits.encode("ascii"))
+    _allowable_chars = frozenset(digits.encode("ascii") + b" ")
 
     def _value_sanitize(self, value):
         value = super(NumericString, self)._value_sanitize(value)
-        if not set(value) <= self.allowable_chars:
+        if not frozenset(value) <= self._allowable_chars:
             raise DecodeError("non-numeric value")
         return value
 
 
-class PrintableString(CommonString):
+class PrintableString(AllowableCharsMixin, CommonString):
+    """Printable string
+
+    Its value is properly sanitized: see X.680 41.4 table 10.
+
+    >>> PrintableString().allowable_chars
+    >>> set([' ', "'", ..., 'z'])
+    """
     __slots__ = ()
     tag_default = tag_encode(19)
     encoding = "ascii"
     asn1_type_name = "PrintableString"
+    _allowable_chars = frozenset(
+        (ascii_letters + digits + " '()+,-./:=?").encode("ascii")
+    )
+
+    def _value_sanitize(self, value):
+        value = super(PrintableString, self)._value_sanitize(value)
+        if not frozenset(value) <= self._allowable_chars:
+            raise DecodeError("non-printable value")
+        return value
 
 
 class TeletexString(CommonString):
@@ -3574,10 +3708,6 @@ class UTCTime(CommonString):
                 self._value = default
 
     def _value_sanitize(self, value):
-        if isinstance(value, self.__class__):
-            return value._value
-        if isinstance(value, datetime):
-            return value.strftime(self.fmt).encode("ascii")
         if isinstance(value, binary_type):
             try:
                 value_decoded = value.decode("ascii")
@@ -3591,6 +3721,10 @@ class UTCTime(CommonString):
                 return value
             else:
                 raise DecodeError("invalid UTCTime length")
+        if isinstance(value, self.__class__):
+            return value._value
+        if isinstance(value, datetime):
+            return value.strftime(self.fmt).encode("ascii")
         raise InvalidValueType((self.__class__, datetime))
 
     def __eq__(self, their):
@@ -3631,6 +3765,7 @@ class UTCTime(CommonString):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -3648,6 +3783,8 @@ class UTCTime(CommonString):
             expl_llen=self.expl_llen if self.expled else None,
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
+            ber_encoded=self.ber_encoded,
+            bered=self.bered,
         )
         for pp in self.pps_lenindef(decode_path):
             yield pp
@@ -3673,12 +3810,6 @@ class GeneralizedTime(UTCTime):
     fmt_ms = "%Y%m%d%H%M%S.%fZ"
 
     def _value_sanitize(self, value):
-        if isinstance(value, self.__class__):
-            return value._value
-        if isinstance(value, datetime):
-            return value.strftime(
-                self.fmt_ms if value.microsecond > 0 else self.fmt
-            ).encode("ascii")
         if isinstance(value, binary_type):
             try:
                 value_decoded = value.decode("ascii")
@@ -3705,6 +3836,12 @@ class GeneralizedTime(UTCTime):
                     "invalid GeneralizedTime length",
                     klass=self.__class__,
                 )
+        if isinstance(value, self.__class__):
+            return value._value
+        if isinstance(value, datetime):
+            return value.strftime(
+                self.fmt_ms if value.microsecond > 0 else self.fmt
+            ).encode("ascii")
         raise InvalidValueType((self.__class__, datetime))
 
     def todatetime(self):
@@ -3830,8 +3967,6 @@ class Choice(Obj):
                 self._value = default_obj.copy()._value
 
     def _value_sanitize(self, value):
-        if isinstance(value, self.__class__):
-            return value._value
         if isinstance(value, tuple) and len(value) == 2:
             choice, obj = value
             spec = self.specs.get(choice)
@@ -3840,12 +3975,21 @@ class Choice(Obj):
             if not isinstance(obj, spec.__class__):
                 raise InvalidValueType((spec,))
             return (choice, spec(obj))
+        if isinstance(value, self.__class__):
+            return value._value
         raise InvalidValueType((self.__class__, tuple))
 
     @property
     def ready(self):
         return self._value is not None and self._value[1].ready
 
+    @property
+    def bered(self):
+        return self.expl_lenindef or (
+            (self._value is not None) and
+            self._value[1].bered
+        )
+
     def copy(self):
         obj = self.__class__(schema=self.specs)
         obj._expl = self._expl
@@ -3854,6 +3998,9 @@ class Choice(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         value = self._value
         if value is not None:
             obj._value = (value[0], value[1].copy())
@@ -3925,7 +4072,7 @@ class Choice(Obj):
         return self._value[1].encode()
 
     def _decode(self, tlv, offset, decode_path, ctx, tag_only):
-        for choice, spec in self.specs.items():
+        for choice, spec in iteritems(self.specs):
             sub_decode_path = decode_path + (choice,)
             try:
                 spec.decode(
@@ -3935,6 +4082,7 @@ class Choice(Obj):
                     decode_path=sub_decode_path,
                     ctx=ctx,
                     tag_only=True,
+                    _ctx_immutable=False,
                 )
             except TagMismatch:
                 continue
@@ -3945,7 +4093,7 @@ class Choice(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
-        if tag_only:
+        if tag_only:  # pragma: no cover
             return
         value, tail = spec.decode(
             tlv,
@@ -3953,6 +4101,7 @@ class Choice(Obj):
             leavemm=True,
             decode_path=sub_decode_path,
             ctx=ctx,
+            _ctx_immutable=False,
         )
         obj = self.__class__(
             schema=self.specs,
@@ -3962,8 +4111,6 @@ class Choice(Obj):
             _decoded=(offset, 0, value.fulllen),
         )
         obj._value = (choice, value)
-        obj.lenindef = value.lenindef
-        obj.bered = value.bered
         return obj, tail
 
     def __repr__(self):
@@ -3974,6 +4121,7 @@ class Choice(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -3987,7 +4135,6 @@ class Choice(Obj):
             llen=self.llen,
             vlen=self.vlen,
             expl_lenindef=self.expl_lenindef,
-            lenindef=self.lenindef,
             bered=self.bered,
         )
         if self.ready:
@@ -4065,18 +4212,26 @@ class Any(Obj):
         self.defined = None
 
     def _value_sanitize(self, value):
+        if isinstance(value, binary_type):
+            return value
         if isinstance(value, self.__class__):
             return value._value
         if isinstance(value, Obj):
             return value.encode()
-        if isinstance(value, binary_type):
-            return value
         raise InvalidValueType((self.__class__, Obj, binary_type))
 
     @property
     def ready(self):
         return self._value is not None
 
+    @property
+    def bered(self):
+        if self.expl_lenindef or self.lenindef:
+            return True
+        if self.defined is None:
+            return False
+        return self.defined[1].bered
+
     def copy(self):
         obj = self.__class__()
         obj._value = self._value
@@ -4086,6 +4241,9 @@ class Any(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         return obj
 
     def __eq__(self, their):
@@ -4149,6 +4307,7 @@ class Any(Obj):
                     decode_path=decode_path + (str(chunk_i),),
                     leavemm=True,
                     ctx=ctx,
+                    _ctx_immutable=False,
                 )
                 vlen += chunk.tlvlen
                 sub_offset += chunk.tlvlen
@@ -4193,6 +4352,7 @@ class Any(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -4211,6 +4371,7 @@ class Any(Obj):
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
             lenindef=self.lenindef,
+            bered=self.bered,
         )
         defined_by, defined = self.defined or (None, None)
         if defined_by is not None:
@@ -4241,8 +4402,7 @@ def get_def_by_path(defines_by_path, sub_decode_path):
 def abs_decode_path(decode_path, rel_path):
     """Create an absolute decode path from current and relative ones
 
-    :param decode_path: current decode path, starting point.
-                        Tuple of strings
+    :param decode_path: current decode path, starting point. Tuple of strings
     :param rel_path: relative path to ``decode_path``. Tuple of strings.
                      If first tuple's element is "/", then treat it as
                      an absolute path, ignoring ``decode_path`` as
@@ -4397,7 +4557,7 @@ class Sequence(Obj):
 
     @property
     def ready(self):
-        for name, spec in self.specs.items():
+        for name, spec in iteritems(self.specs):
             value = self._value.get(name)
             if value is None:
                 if spec.optional:
@@ -4408,6 +4568,12 @@ class Sequence(Obj):
                     return False
         return True
 
+    @property
+    def bered(self):
+        if self.expl_lenindef or self.lenindef or self.ber_encoded:
+            return True
+        return any(value.bered for value in itervalues(self._value))
+
     def copy(self):
         obj = self.__class__(schema=self.specs)
         obj.tag = self.tag
@@ -4417,7 +4583,10 @@ class Sequence(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
-        obj._value = {k: v.copy() for k, v in self._value.items()}
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
+        obj._value = {k: v.copy() for k, v in iteritems(self._value)}
         return obj
 
     def __eq__(self, their):
@@ -4478,7 +4647,7 @@ class Sequence(Obj):
 
     def _encoded_values(self):
         raws = []
-        for name, spec in self.specs.items():
+        for name, spec in iteritems(self.specs):
             value = self._value.get(name)
             if value is None:
                 if spec.optional:
@@ -4507,7 +4676,7 @@ class Sequence(Obj):
                 decode_path=decode_path,
                 offset=offset,
             )
-        if tag_only:
+        if tag_only:  # pragma: no cover
             return
         lenindef = False
         ctx_bered = ctx.get("bered", False)
@@ -4542,9 +4711,9 @@ class Sequence(Obj):
         vlen = 0
         sub_offset = offset + tlen + llen
         values = {}
-        bered = False
+        ber_encoded = False
         ctx_allow_default_values = ctx.get("allow_default_values", False)
-        for name, spec in self.specs.items():
+        for name, spec in iteritems(self.specs):
             if spec.optional and (
                     (lenindef and v[:EOC_LEN].tobytes() == EOC) or
                     len(v) == 0
@@ -4558,6 +4727,7 @@ class Sequence(Obj):
                     leavemm=True,
                     decode_path=sub_decode_path,
                     ctx=ctx,
+                    _ctx_immutable=False,
                 )
             except TagMismatch:
                 if spec.optional:
@@ -4582,6 +4752,7 @@ class Sequence(Obj):
                             leavemm=True,
                             decode_path=sub_sub_decode_path,
                             ctx=ctx,
+                            _ctx_immutable=False,
                         )
                         if len(defined_tail) > 0:
                             raise DecodeError(
@@ -4601,6 +4772,7 @@ class Sequence(Obj):
                         leavemm=True,
                         decode_path=sub_decode_path + (DecodePathDefBy(defined_by),),
                         ctx=ctx,
+                        _ctx_immutable=False,
                     )
                     if len(defined_tail) > 0:
                         raise DecodeError(
@@ -4617,7 +4789,7 @@ class Sequence(Obj):
             v = v_tail
             if spec.default is not None and value == spec.default:
                 if ctx_bered or ctx_allow_default_values:
-                    bered = True
+                    ber_encoded = True
                 else:
                     raise DecodeError(
                         "DEFAULT value met",
@@ -4667,7 +4839,7 @@ class Sequence(Obj):
         )
         obj._value = values
         obj.lenindef = lenindef
-        obj.bered = bered
+        obj.ber_encoded = ber_encoded
         return obj, tail
 
     def __repr__(self):
@@ -4682,6 +4854,7 @@ class Sequence(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -4699,6 +4872,8 @@ class Sequence(Obj):
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
             lenindef=self.lenindef,
+            ber_encoded=self.ber_encoded,
+            bered=self.bered,
         )
         for name in self.specs:
             value = self._value.get(name)
@@ -4732,6 +4907,9 @@ class Set(Sequence):
         v = b"".join(raws)
         return b"".join((self.tag, len_encode(len(v)), v))
 
+    def _specs_items(self):
+        return iteritems(self.specs)
+
     def _decode(self, tlv, offset, decode_path, ctx, tag_only):
         try:
             t, tlen, lv = tag_strip(tlv)
@@ -4782,15 +4960,15 @@ class Set(Sequence):
         vlen = 0
         sub_offset = offset + tlen + llen
         values = {}
-        bered = False
+        ber_encoded = False
         ctx_allow_default_values = ctx.get("allow_default_values", False)
         ctx_allow_unordered_set = ctx.get("allow_unordered_set", False)
         value_prev = memoryview(v[:0])
-        specs_items = self.specs.items
+
         while len(v) > 0:
             if lenindef and v[:EOC_LEN].tobytes() == EOC:
                 break
-            for name, spec in specs_items():
+            for name, spec in self._specs_items():
                 sub_decode_path = decode_path + (name,)
                 try:
                     spec.decode(
@@ -4800,6 +4978,7 @@ class Set(Sequence):
                         decode_path=sub_decode_path,
                         ctx=ctx,
                         tag_only=True,
+                        _ctx_immutable=False,
                     )
                 except TagMismatch:
                     continue
@@ -4816,11 +4995,12 @@ class Set(Sequence):
                 leavemm=True,
                 decode_path=sub_decode_path,
                 ctx=ctx,
+                _ctx_immutable=False,
             )
             value_len = value.fulllen
             if value_prev.tobytes() > v[:value_len].tobytes():
                 if ctx_bered or ctx_allow_unordered_set:
-                    bered = True
+                    ber_encoded = True
                 else:
                     raise DecodeError(
                         "unordered " + self.asn1_type_name,
@@ -4831,7 +5011,7 @@ class Set(Sequence):
             if spec.default is None or value != spec.default:
                 pass
             elif ctx_bered or ctx_allow_default_values:
-                bered = True
+                ber_encoded = True
             else:
                 raise DecodeError(
                     "DEFAULT value met",
@@ -4870,7 +5050,7 @@ class Set(Sequence):
                 decode_path=decode_path,
                 offset=offset,
             )
-        obj.bered = bered
+        obj.ber_encoded = ber_encoded
         return obj, tail
 
 
@@ -4969,6 +5149,12 @@ class SequenceOf(Obj):
     def ready(self):
         return all(v.ready for v in self._value)
 
+    @property
+    def bered(self):
+        if self.expl_lenindef or self.lenindef or self.ber_encoded:
+            return True
+        return any(v.bered for v in self._value)
+
     def copy(self):
         obj = self.__class__(schema=self.spec)
         obj._bound_min = self._bound_min
@@ -4980,6 +5166,9 @@ class SequenceOf(Obj):
         obj.offset = self.offset
         obj.llen = self.llen
         obj.vlen = self.vlen
+        obj.expl_lenindef = self.expl_lenindef
+        obj.lenindef = self.lenindef
+        obj.ber_encoded = self.ber_encoded
         obj._value = [v.copy() for v in self._value]
         return obj
 
@@ -5107,7 +5296,7 @@ class SequenceOf(Obj):
         _value = []
         ctx_allow_unordered_set = ctx.get("allow_unordered_set", False)
         value_prev = memoryview(v[:0])
-        bered = False
+        ber_encoded = False
         spec = self.spec
         while len(v) > 0:
             if lenindef and v[:EOC_LEN].tobytes() == EOC:
@@ -5119,12 +5308,13 @@ class SequenceOf(Obj):
                 leavemm=True,
                 decode_path=sub_decode_path,
                 ctx=ctx,
+                _ctx_immutable=False,
             )
             value_len = value.fulllen
             if ordering_check:
                 if value_prev.tobytes() > v[:value_len].tobytes():
                     if ctx_bered or ctx_allow_unordered_set:
-                        bered = True
+                        ber_encoded = True
                     else:
                         raise DecodeError(
                             "unordered " + self.asn1_type_name,
@@ -5165,7 +5355,7 @@ class SequenceOf(Obj):
                 )
             obj.lenindef = True
             tail = v[EOC_LEN:]
-        obj.bered = bered
+        obj.ber_encoded = ber_encoded
         return obj, tail
 
     def __repr__(self):
@@ -5176,6 +5366,7 @@ class SequenceOf(Obj):
 
     def pps(self, decode_path=()):
         yield _pp(
+            obj=self,
             asn1_type_name=self.asn1_type_name,
             obj_name=self.__class__.__name__,
             decode_path=decode_path,
@@ -5193,6 +5384,8 @@ class SequenceOf(Obj):
             expl_vlen=self.expl_vlen if self.expled else None,
             expl_lenindef=self.expl_lenindef,
             lenindef=self.lenindef,
+            ber_encoded=self.ber_encoded,
+            bered=self.bered,
         )
         for i, value in enumerate(self._value):
             yield value.pps(decode_path=decode_path + (str(i),))
@@ -5249,7 +5442,7 @@ def generic_decoder():  # pragma: no cover
     choice = PrimitiveTypes()
     choice.specs["SequenceOf"] = SequenceOf(schema=choice)
     choice.specs["SetOf"] = SetOf(schema=choice)
-    for i in range(31):
+    for i in six_xrange(31):
         choice.specs["SequenceOf%d" % i] = SequenceOf(
             schema=choice,
             expl=tag_ctxc(i),