mmap-ed memoryview support

diff --git a/doc/news.rst b/doc/news.rst
index 2419f8ef02287ad7ba8c80f330346e67a6debfde..7e110786a3ff064d001a0a00c2e920f34558c833 100644 (file)
--- a/doc/news.rst
+++ b/doc/news.rst
@@ -18,6 +18,7 @@ News
   ASN.1 data without fully parsing it first
 * Initial experimental CER encoding mode, allowing streaming encoding of
   the data directly to some writeable object
+* Ability to use mmap-ed memoryviews to skip files loading to memory
 
 .. _release6.3:
 

diff --git a/pyderasn.py b/pyderasn.py
index 3533c6b9f18f46f28d9812b1dce8824b29ecfa44..002f0605c42e65b804ffc1d4907142a7000cd521 100755 (executable)
--- a/pyderasn.py
+++ b/pyderasn.py
@@ -644,6 +644,7 @@ Various
 .. autofunction:: pyderasn.abs_decode_path
 .. autofunction:: pyderasn.colonize_hex
 .. autofunction:: pyderasn.encode_cer
+.. autofunction:: pyderasn.file_mmaped
 .. autofunction:: pyderasn.hexenc
 .. autofunction:: pyderasn.hexdec
 .. autofunction:: pyderasn.tag_encode
@@ -782,6 +783,8 @@ from datetime import datetime
 from datetime import timedelta
 from io import BytesIO
 from math import ceil
+from mmap import mmap
+from mmap import PROT_READ
 from operator import attrgetter
 from string import ascii_letters
 from string import digits
@@ -824,6 +827,7 @@ __all__ = (
     "encode_cer",
     "Enumerated",
     "ExceedingData",
+    "file_mmaped",
     "GeneralizedTime",
     "GeneralString",
     "GraphicString",
@@ -891,6 +895,14 @@ DECIMALS = frozenset(digits)
 DECIMAL_SIGNS = ".,"
 
 
+def file_mmaped(fd):
+    """Make mmap-ed memoryview for reading from file
+
+    :param fd: file object
+    :returns: memoryview over read-only mmap-ing of the whole file
+    """
+    return memoryview(mmap(fd.fileno(), 0, prot=PROT_READ))
+
 def pureint(value):
     if not set(value) <= DECIMALS:
         raise ValueError("non-pure integer")
@@ -6695,14 +6707,17 @@ def main():  # pragma: no cover
         help="Allow explicit tag out-of-bound",
     )
     parser.add_argument(
-        "DERFile",
+        "RAWFile",
         type=argparse.FileType("rb"),
-        help="Path to DER file you want to decode",
+        help="Path to BER/CER/DER file you want to decode",
     )
     args = parser.parse_args()
-    args.DERFile.seek(args.skip)
-    der = memoryview(args.DERFile.read())
-    args.DERFile.close()
+    if PY2:
+        args.RAWFile.seek(args.skip)
+        raw = memoryview(args.RAWFile.read())
+        args.RAWFile.close()
+    else:
+        raw = file_mmaped(args.RAWFile)[args.skip:]
     oid_maps = (
         [obj_by_path(_path) for _path in (args.oids or "").split(",")]
         if args.oids else ()
@@ -6719,7 +6734,7 @@ def main():  # pragma: no cover
     }
     if args.defines_by_path is not None:
         ctx["defines_by_path"] = obj_by_path(args.defines_by_path)
-    obj, tail = schema().decode(der, ctx=ctx)
+    obj, tail = schema().decode(raw, ctx=ctx)
     from os import environ
     print(pprinter(
         obj,

diff --git a/tests/test_crl.py b/tests/test_crl.py
index 317355213365e389c5d17f5bc3a31857df8e3dd2..70692303726d28af6bb5b38292c49f146dabf75b 100644 (file)
--- a/tests/test_crl.py
+++ b/tests/test_crl.py
@@ -22,8 +22,11 @@ from time import time
 from unittest import skipIf
 from unittest import TestCase
 
+from six import PY2
+
 from pyderasn import BitString
 from pyderasn import encode_cer
+from pyderasn import file_mmaped
 from pyderasn import Sequence
 from pyderasn import SequenceOf
 from pyderasn import tag_ctxc
@@ -87,3 +90,10 @@ class TestCACert(TestCase):
         der_raw = crl2.encode()
         print("DER encoded", time() - start)
         self.assertSequenceEqual(der_raw, raw)
+
+    @skipIf(PY2, "Py27 mmap does not implement buffer protocol")
+    def test_mmaped(self):
+        fd = open("revoke.crl", "rb")
+        start = time()
+        CertificateList().decod(file_mmaped(fd))
+        print("DER decoded", time() - start)