Replace zlib with zstd

diff --git a/doc/cmds.texi b/doc/cmds.texi
index c2107bed94fe2f1157925104767b5899de82b86d..d5395238b7f58fe3f4c205005ea1da3d3175fbba 100644 (file)
--- a/doc/cmds.texi
+++ b/doc/cmds.texi
@@ -387,7 +387,7 @@ Path: stargrave@stargrave.org
 
 And with the @option{-dump} option it will give you the actual payload
 (the whole file, mail message, and so on). @option{-decompress} option
-tries to zlib-decompress the data from plain packet (useful for mail
+tries to zstd-decompress the data from plain packet (useful for mail
 packets).
 
 @option{-overheads} options print encrypted, plain and size header overheads.

diff --git a/doc/download.texi b/doc/download.texi
index f675b2d772a6021875f330e8244b60c6bc09515b..f25c62e6b268b6aa922c3a6ae99213c743ddf5fe 100644 (file)
--- a/doc/download.texi
+++ b/doc/download.texi
@@ -13,6 +13,7 @@ Tarballs include all necessary required libraries:
 @item @code{github.com/flynn/noise} @tab BSD 3-Clause
 @item @code{github.com/gorhill/cronexpr} @tab GNU GPLv3
 @item @code{github.com/hjson/hjson-go} @tab MIT
+@item @code{github.com/klauspost/compress} @tab BSD 3-Clause
 @item @code{go.cypherpunks.ru/balloon} @tab GNU LGPLv3
 @item @code{golang.org/x/crypto} @tab BSD 3-Clause
 @item @code{golang.org/x/net} @tab BSD 3-Clause

diff --git a/doc/news.ru.texi b/doc/news.ru.texi
index c9bd97f963b050fa4da7bdec0794ef427ec80f7e..d6047c8a180f594eb54fff85836f19840f824aa9 100644 (file)
--- a/doc/news.ru.texi
+++ b/doc/news.ru.texi
@@ -7,6 +7,10 @@
 @item @strong{Несовместимое} изменение формата конфигурационного файла:
     YAML заменён на Hjson, из-за его гораздо большей простоты, без
     заметного потеря функционала и удобства.
+@item @strong{Несовместимое} изменение формата простых пакетов. Работа
+    со старыми версиями не поддерживается. @code{zlib} сжатие заменено
+    на @code{Zstandard}, так как оно значительно быстрее и эффективнее,
+    не смотря на то, что версия библиотеки ещё не проверена временем.
 @item @command{nncp-cfgnew} генерирует конфигурационный файл с
     множеством комментариев. Можно использовать @option{-nocomments}
     опцию для старого поведения.

diff --git a/doc/news.texi b/doc/news.texi
index 211f399f5085f394703321233c7493265298bcba..d26d9a9a9489521e616bf88f45322cdc27629296 100644 (file)
--- a/doc/news.texi
+++ b/doc/news.texi
@@ -9,6 +9,10 @@ See also this page @ref{Новости, on russian}.
 @item @strong{Incompatible} configuration file format change: YAML is
     replaced with Hjson, due to its simplicity, without noticeable lack
     of either functionality or convenience.
+@strong{Incompatible} plain packet format changes. Older versions are
+    not supported. @code{zlib} compression is replaced with
+    @code{Zstandard}, due to its speed and efficiency, however library
+    version is not mature enough.
 @item @command{nncp-cfgnew} generates configuration file with many
     comments. @option{-nocomments} option can be used for an old
     behaviour.

diff --git a/doc/pkt.texi b/doc/pkt.texi
index d216f93e96d9d05e614c30f9f39a343b833ab552..f7bbe564c5b9ef7702177055af7e23ae3f2e914c 100644 (file)
--- a/doc/pkt.texi
+++ b/doc/pkt.texi
@@ -28,7 +28,7 @@ drive.
 @headitem @tab XDR type @tab Value
 @item Magic number @tab
     8-byte, fixed length opaque data @tab
-    @verb{|N N C P P 0x00 0x00 0x02|}
+    @verb{|N N C P P 0x00 0x00 0x03|}
 @item Payload type @tab
     unsigned integer @tab
     0 (file), 1 (freq), 2 (exec), 3 (transition)
@@ -58,7 +58,7 @@ Depending on the packet's type, payload could store:
 @itemize
 @item File contents
 @item Destination path for freq
-@item @url{http://zlib.net/, zlib} compressed exec body
+@item @url{https://facebook.github.io/zstd/, Zstandard} compressed exec body
 @item Whole encrypted packet we need to relay on
 @end itemize
 

diff --git a/doc/sources.texi b/doc/sources.texi
index f74985fa74824752fc70d5c9679ad54ae16a96cc..2ded27ca827a3286b2eac472973d23b2115498c4 100644 (file)
--- a/doc/sources.texi
+++ b/doc/sources.texi
@@ -20,8 +20,10 @@ repositories will be unavailable (they are seldom updated):
 @item @code{github.com/davecgh/go-xdr} @tab @url{git://git.cypherpunks.ru/go-xdr.git}
 @item @code{github.com/dustin/go-humanize} @tab @url{git://git.cypherpunks.ru/go-humanize.git}
 @item @code{github.com/flynn/noise} @tab @url{git://git.cypherpunks.ru/noise.git}
+@item @code{github.com/google/go-cmp} @tab @url{git://git.cypherpunks.ru/go-cmp.git}
 @item @code{github.com/gorhill/cronexpr} @tab @url{git://git.cypherpunks.ru/cronexpr.git}
 @item @code{github.com/hjson/hjson-go} @tab @url{git://git.cypherpunks.ru/hjson-go.git}
+@item @code{github.com/klauspost/compress} @tab @url{git://git.cypherpunks.ru/compress.git}
 @item @code{golang.org/x/crypto} @tab @url{git://git.cypherpunks.ru/crypto.git}
 @item @code{golang.org/x/net} @tab @url{git://git.cypherpunks.ru/net.git}
 @item @code{golang.org/x/sys} @tab @url{git://git.cypherpunks.ru/sys.git}

diff --git a/makedist.sh b/makedist.sh
index 311c2daec523c8e535baaf25c1419cc06a091ff9..70b75159b1a11c7d660fef7107951d395b23acc4 100755 (executable)
--- a/makedist.sh
+++ b/makedist.sh
@@ -22,6 +22,7 @@ github.com/dustin/go-humanize
 github.com/flynn/noise
 github.com/gorhill/cronexpr
 github.com/hjson/hjson-go
+github.com/klauspost/compress
 go.cypherpunks.ru/balloon
 golang.org/x/crypto
 golang.org/x/net
@@ -75,6 +76,23 @@ tar cfCI - src $tmp/includes | tar xfC - $tmp
 rm -fr src/golang.org $tmp/includes
 mv $tmp/golang.org src
 
+cat > $tmp/includes <<EOF
+compress/compressible.go
+compress/fse
+compress/huff0
+compress/LICENSE
+compress/README.md
+compress/zstd
+EOF
+cat > $tmp/excludes <<EOF
+*testdata*
+*_test.go
+snappy.go
+EOF
+tar cfCIX - src/github.com/klauspost $tmp/includes $tmp/excludes | tar xfC - $tmp
+rm -fr src/github.com/klauspost/compress $tmp/includes $tmp/excludes
+mv $tmp/compress src/github.com/klauspost
+
 find src -name .travis.yml -delete
 rm -fr src/github.com/davecgh/go-xdr/xdr
 rm -r src/github.com/flynn/noise/vector*

diff --git a/src/cmd/nncp-exec/main.go b/src/cmd/nncp-exec/main.go
index 0a02252f105ba42f16ebb40f78ebc048e0c2379f..39de8ab12a3f2fed180ecd2b89cdab763cfe8e07 100644 (file)
--- a/src/cmd/nncp-exec/main.go
+++ b/src/cmd/nncp-exec/main.go
@@ -22,7 +22,6 @@ import (
        "bufio"
        "flag"
        "fmt"
-       "io/ioutil"
        "log"
        "os"
 
@@ -88,18 +87,13 @@ func main() {
 
        nncp.ViaOverride(*viaOverride, ctx, node)
 
-       body, err := ioutil.ReadAll(bufio.NewReader(os.Stdin))
-       if err != nil {
-               log.Fatalln("Can not read body from stdin:", err)
-       }
-
        if err = ctx.TxExec(
                node,
                nice,
                replyNice,
                flag.Args()[1],
                flag.Args()[2:],
-               body,
+               bufio.NewReader(os.Stdin),
                int64(*minSize)*1024,
        ); err != nil {
                log.Fatalln(err)

diff --git a/src/cmd/nncp-pkt/main.go b/src/cmd/nncp-pkt/main.go
index 82da1d35b7673e9fd9b340aae77291e4a9abfa71..1808e3a61d001d11bf49d3120fe317e70bb48c86 100644 (file)
--- a/src/cmd/nncp-pkt/main.go
+++ b/src/cmd/nncp-pkt/main.go
@@ -21,7 +21,6 @@ package main
 import (
        "bufio"
        "bytes"
-       "compress/zlib"
        "flag"
        "fmt"
        "io"
@@ -29,6 +28,7 @@ import (
        "os"
 
        "github.com/davecgh/go-xdr/xdr2"
+       "github.com/klauspost/compress/zstd"
        "go.cypherpunks.ru/nncp/v5"
 )
 
@@ -44,7 +44,7 @@ func main() {
        var (
                overheads  = flag.Bool("overheads", false, "Print packet overheads")
                dump       = flag.Bool("dump", false, "Write decrypted/parsed payload to stdout")
-               decompress = flag.Bool("decompress", false, "Try to zlib decompress dumped data")
+               decompress = flag.Bool("decompress", false, "Try to zstd decompress dumped data")
                cfgPath    = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file")
                version    = flag.Bool("version", false, "Print version information")
                warranty   = flag.Bool("warranty", false, "Print warranty information")
@@ -77,13 +77,13 @@ func main() {
        }
        var pkt nncp.Pkt
        _, err = xdr.Unmarshal(bytes.NewReader(beginning), &pkt)
-       if err == nil && pkt.Magic == nncp.MagicNNCPPv2 {
+       if err == nil && pkt.Magic == nncp.MagicNNCPPv3 {
                if *dump {
                        bufW := bufio.NewWriter(os.Stdout)
                        var r io.Reader
                        r = bufio.NewReader(os.Stdin)
                        if *decompress {
-                               decompressor, err := zlib.NewReader(r)
+                               decompressor, err := zstd.NewReader(r)
                                if err != nil {
                                        log.Fatalln(err)
                                }

diff --git a/src/go.mod b/src/go.mod
index 0ce4d58da9f425d8140f115e5ed64080822525a7..b15af4159caec16b42fae53d6bf92395cf6d266f 100644 (file)
--- a/src/go.mod
+++ b/src/go.mod
@@ -4,8 +4,10 @@ require (
        github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892
        github.com/dustin/go-humanize v1.0.0
        github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6
+       github.com/google/go-cmp v0.3.1 // indirect
        github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
        github.com/hjson/hjson-go v3.0.1+incompatible
+       github.com/klauspost/compress v1.9.2
        github.com/kr/pretty v0.1.0 // indirect
        go.cypherpunks.ru/balloon v1.1.0
        golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc

diff --git a/src/go.sum b/src/go.sum
index 8fa6343b06b3f656cd498aaebd1602ac402a3332..7c2b3c9e6f0aa127c6c2ebacc15743567b669e45 100644 (file)
--- a/src/go.sum
+++ b/src/go.sum
@@ -4,10 +4,14 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY=
 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA=
 github.com/hjson/hjson-go v3.0.1+incompatible h1:JwOXblcMiBbiWue7iPkoFK9oXSnW8n+qXh/0Fio6TCo=
 github.com/hjson/hjson-go v3.0.1+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
+github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY=
+github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

diff --git a/src/pkt.go b/src/pkt.go
index cb75c237286383bbd0962cbfdeb01e0445c55acc..2fab844422486793be179dd571ef5f77386a3e1c 100644 (file)
--- a/src/pkt.go
+++ b/src/pkt.go
@@ -51,7 +51,7 @@ const (
 )
 
 var (
-       MagicNNCPPv2 [8]byte = [8]byte{'N', 'N', 'C', 'P', 'P', 0, 0, 2}
+       MagicNNCPPv3 [8]byte = [8]byte{'N', 'N', 'C', 'P', 'P', 0, 0, 3}
        MagicNNCPEv4 [8]byte = [8]byte{'N', 'N', 'C', 'P', 'E', 0, 0, 4}
        BadMagic     error   = errors.New("Unknown magic number")
        BadPktType   error   = errors.New("Unknown packet type")
@@ -123,7 +123,7 @@ func NewPkt(typ PktType, nice uint8, path []byte) (*Pkt, error) {
                return nil, errors.New("Too long path")
        }
        pkt := Pkt{
-               Magic:   MagicNNCPPv2,
+               Magic:   MagicNNCPPv3,
                Type:    typ,
                Nice:    nice,
                PathLen: uint8(len(path)),

diff --git a/src/toss.go b/src/toss.go
index 2c4726f8cb7dc7fc2e94334fe8b97c1d3363eac6..0ce446ac807b39df6e6a067a2745b0631101ede2 100644 (file)
--- a/src/toss.go
+++ b/src/toss.go
@@ -20,7 +20,6 @@ package nncp
 import (
        "bufio"
        "bytes"
-       "compress/zlib"
        "fmt"
        "io"
        "io/ioutil"
@@ -35,6 +34,7 @@ import (
 
        "github.com/davecgh/go-xdr/xdr2"
        "github.com/dustin/go-humanize"
+       "github.com/klauspost/compress/zstd"
        "golang.org/x/crypto/blake2b"
        "golang.org/x/crypto/poly1305"
 )
@@ -58,6 +58,11 @@ func (ctx *Ctx) Toss(
        dryRun, doSeen, noFile, noFreq, noExec, noTrns bool,
 ) bool {
        isBad := false
+       decompressor, err := zstd.NewReader(nil)
+       if err != nil {
+               panic(err)
+       }
+       defer decompressor.Close()
        for job := range ctx.Jobs(nodeId, TRx) {
                pktName := filepath.Base(job.Fd.Name())
                sds := SDS{"node": job.PktEnc.Sender, "pkt": pktName}
@@ -117,10 +122,6 @@ func (ctx *Ctx) Toss(
                                "type": "exec",
                                "dst":  strings.Join(append([]string{handle}, args...), " "),
                        })
-                       decompressor, err := zlib.NewReader(pipeR)
-                       if err != nil {
-                               log.Fatalln(err)
-                       }
                        sender := ctx.Neigh[*job.PktEnc.Sender]
                        cmdline, exists := sender.Exec[handle]
                        if !exists || len(cmdline) == 0 {
@@ -128,6 +129,9 @@ func (ctx *Ctx) Toss(
                                isBad = true
                                goto Closing
                        }
+                       if err = decompressor.Reset(pipeR); err != nil {
+                               log.Fatalln(err)
+                       }
                        if !dryRun {
                                cmd := exec.Command(
                                        cmdline[0],

diff --git a/src/toss_test.go b/src/toss_test.go
index a96f78bd68afb6f28141ee361e37eaa1e802a4be..4ccf4ead4ca656ff8738b6773b5cf385b56339ae 100644 (file)
--- a/src/toss_test.go
+++ b/src/toss_test.go
@@ -26,6 +26,7 @@ import (
        "os"
        "path/filepath"
        "strconv"
+       "strings"
        "testing"
        "testing/quick"
 
@@ -96,7 +97,7 @@ func TestTossExec(t *testing.T) {
                                replyNice,
                                handle,
                                []string{"arg0", "arg1"},
-                               []byte("BODY\n"),
+                               strings.NewReader("BODY\n"),
                                1<<15,
                        ); err != nil {
                                t.Error(err)
@@ -441,7 +442,7 @@ func TestTossTrns(t *testing.T) {
                os.MkdirAll(txPath, os.FileMode(0700))
                for _, data := range datum {
                        pktTrans := Pkt{
-                               Magic:   MagicNNCPPv2,
+                               Magic:   MagicNNCPPv3,
                                Type:    PktTypeTrns,
                                PathLen: blake2b.Size256,
                                Path:    new([MaxPathSize]byte),

diff --git a/src/tx.go b/src/tx.go
index cb8153fe0419705b991c621a662396e1238c5e64..a509207422ccee4479b514b77c70736acc86e018 100644 (file)
--- a/src/tx.go
+++ b/src/tx.go
@@ -20,7 +20,6 @@ package nncp
 import (
        "bufio"
        "bytes"
-       "compress/zlib"
        "crypto/rand"
        "errors"
        "hash"
@@ -32,6 +31,7 @@ import (
        "strings"
 
        "github.com/davecgh/go-xdr/xdr2"
+       "github.com/klauspost/compress/zstd"
        "golang.org/x/crypto/blake2b"
        "golang.org/x/crypto/chacha20poly1305"
 )
@@ -381,7 +381,7 @@ func (ctx *Ctx) TxExec(
        nice, replyNice uint8,
        handle string,
        args []string,
-       body []byte,
+       in io.Reader,
        minSize int64,
 ) error {
        path := make([][]byte, 0, 1+len(args))
@@ -394,14 +394,18 @@ func (ctx *Ctx) TxExec(
                return err
        }
        var compressed bytes.Buffer
-       compressor, err := zlib.NewWriterLevel(&compressed, zlib.BestCompression)
+       compressor, err := zstd.NewWriter(
+               &compressed,
+               zstd.WithEncoderLevel(zstd.SpeedDefault),
+       )
        if err != nil {
                return err
        }
-       if _, err = io.Copy(compressor, bytes.NewReader(body)); err != nil {
+       _, err = io.Copy(compressor, in)
+       compressor.Close()
+       if err != nil {
                return err
        }
-       compressor.Close()
        size := int64(compressed.Len())
        _, err = ctx.Tx(node, pkt, nice, size, minSize, &compressed)
        sds := SDS{