From: Sergey Matveev <stargrave@stargrave.org>
Date: Wed, 13 Nov 2019 14:25:06 +0000 (+0300)
Subject: Replace zlib with zstd
X-Git-Tag: v5.0.0^2~7
X-Git-Url: http://www.git.cypherpunks.ru/?p=nncp.git;a=commitdiff_plain;h=9c25694667c14c934fc122741b9736988172f7de

Replace zlib with zstd
---

diff --git a/doc/cmds.texi b/doc/cmds.texi
index c2107be..d539523 100644
--- a/doc/cmds.texi
+++ b/doc/cmds.texi
@@ -387,7 +387,7 @@ Path: stargrave@stargrave.org
 
 And with the @option{-dump} option it will give you the actual payload
 (the whole file, mail message, and so on). @option{-decompress} option
-tries to zlib-decompress the data from plain packet (useful for mail
+tries to zstd-decompress the data from plain packet (useful for mail
 packets).
 
 @option{-overheads} options print encrypted, plain and size header overheads.
diff --git a/doc/download.texi b/doc/download.texi
index f675b2d..f25c62e 100644
--- a/doc/download.texi
+++ b/doc/download.texi
@@ -13,6 +13,7 @@ Tarballs include all necessary required libraries:
 @item @code{github.com/flynn/noise} @tab BSD 3-Clause
 @item @code{github.com/gorhill/cronexpr} @tab GNU GPLv3
 @item @code{github.com/hjson/hjson-go} @tab MIT
+@item @code{github.com/klauspost/compress} @tab BSD 3-Clause
 @item @code{go.cypherpunks.ru/balloon} @tab GNU LGPLv3
 @item @code{golang.org/x/crypto} @tab BSD 3-Clause
 @item @code{golang.org/x/net} @tab BSD 3-Clause
diff --git a/doc/news.ru.texi b/doc/news.ru.texi
index c9bd97f..d6047c8 100644
--- a/doc/news.ru.texi
+++ b/doc/news.ru.texi
@@ -7,6 +7,10 @@
 @item @strong{Несовместимое} изменение формата конфигурационного файла:
     YAML заменён на Hjson, из-за его гораздо большей простоты, без
     заметного потеря функционала и удобства.
+@item @strong{Несовместимое} изменение формата простых пакетов. Работа
+    со старыми версиями не поддерживается. @code{zlib} сжатие заменено
+    на @code{Zstandard}, так как оно значительно быстрее и эффективнее,
+    не смотря на то, что версия библиотеки ещё не проверена временем.
 @item @command{nncp-cfgnew} генерирует конфигурационный файл с
     множеством комментариев. Можно использовать @option{-nocomments}
     опцию для старого поведения.
diff --git a/doc/news.texi b/doc/news.texi
index 211f399..d26d9a9 100644
--- a/doc/news.texi
+++ b/doc/news.texi
@@ -9,6 +9,10 @@ See also this page @ref{Новости, on russian}.
 @item @strong{Incompatible} configuration file format change: YAML is
     replaced with Hjson, due to its simplicity, without noticeable lack
     of either functionality or convenience.
+@strong{Incompatible} plain packet format changes. Older versions are
+    not supported. @code{zlib} compression is replaced with
+    @code{Zstandard}, due to its speed and efficiency, however library
+    version is not mature enough.
 @item @command{nncp-cfgnew} generates configuration file with many
     comments. @option{-nocomments} option can be used for an old
     behaviour.
diff --git a/doc/pkt.texi b/doc/pkt.texi
index d216f93..f7bbe56 100644
--- a/doc/pkt.texi
+++ b/doc/pkt.texi
@@ -28,7 +28,7 @@ drive.
 @headitem @tab XDR type @tab Value
 @item Magic number @tab
     8-byte, fixed length opaque data @tab
-    @verb{|N N C P P 0x00 0x00 0x02|}
+    @verb{|N N C P P 0x00 0x00 0x03|}
 @item Payload type @tab
     unsigned integer @tab
     0 (file), 1 (freq), 2 (exec), 3 (transition)
@@ -58,7 +58,7 @@ Depending on the packet's type, payload could store:
 @itemize
 @item File contents
 @item Destination path for freq
-@item @url{http://zlib.net/, zlib} compressed exec body
+@item @url{https://facebook.github.io/zstd/, Zstandard} compressed exec body
 @item Whole encrypted packet we need to relay on
 @end itemize
 
diff --git a/doc/sources.texi b/doc/sources.texi
index f74985f..2ded27c 100644
--- a/doc/sources.texi
+++ b/doc/sources.texi
@@ -20,8 +20,10 @@ repositories will be unavailable (they are seldom updated):
 @item @code{github.com/davecgh/go-xdr} @tab @url{git://git.cypherpunks.ru/go-xdr.git}
 @item @code{github.com/dustin/go-humanize} @tab @url{git://git.cypherpunks.ru/go-humanize.git}
 @item @code{github.com/flynn/noise} @tab @url{git://git.cypherpunks.ru/noise.git}
+@item @code{github.com/google/go-cmp} @tab @url{git://git.cypherpunks.ru/go-cmp.git}
 @item @code{github.com/gorhill/cronexpr} @tab @url{git://git.cypherpunks.ru/cronexpr.git}
 @item @code{github.com/hjson/hjson-go} @tab @url{git://git.cypherpunks.ru/hjson-go.git}
+@item @code{github.com/klauspost/compress} @tab @url{git://git.cypherpunks.ru/compress.git}
 @item @code{golang.org/x/crypto} @tab @url{git://git.cypherpunks.ru/crypto.git}
 @item @code{golang.org/x/net} @tab @url{git://git.cypherpunks.ru/net.git}
 @item @code{golang.org/x/sys} @tab @url{git://git.cypherpunks.ru/sys.git}
diff --git a/makedist.sh b/makedist.sh
index 311c2da..70b7515 100755
--- a/makedist.sh
+++ b/makedist.sh
@@ -22,6 +22,7 @@ github.com/dustin/go-humanize
 github.com/flynn/noise
 github.com/gorhill/cronexpr
 github.com/hjson/hjson-go
+github.com/klauspost/compress
 go.cypherpunks.ru/balloon
 golang.org/x/crypto
 golang.org/x/net
@@ -75,6 +76,23 @@ tar cfCI - src $tmp/includes | tar xfC - $tmp
 rm -fr src/golang.org $tmp/includes
 mv $tmp/golang.org src
 
+cat > $tmp/includes <<EOF
+compress/compressible.go
+compress/fse
+compress/huff0
+compress/LICENSE
+compress/README.md
+compress/zstd
+EOF
+cat > $tmp/excludes <<EOF
+*testdata*
+*_test.go
+snappy.go
+EOF
+tar cfCIX - src/github.com/klauspost $tmp/includes $tmp/excludes | tar xfC - $tmp
+rm -fr src/github.com/klauspost/compress $tmp/includes $tmp/excludes
+mv $tmp/compress src/github.com/klauspost
+
 find src -name .travis.yml -delete
 rm -fr src/github.com/davecgh/go-xdr/xdr
 rm -r src/github.com/flynn/noise/vector*
diff --git a/src/cmd/nncp-exec/main.go b/src/cmd/nncp-exec/main.go
index 0a02252..39de8ab 100644
--- a/src/cmd/nncp-exec/main.go
+++ b/src/cmd/nncp-exec/main.go
@@ -22,7 +22,6 @@ import (
 	"bufio"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 
@@ -88,18 +87,13 @@ func main() {
 
 	nncp.ViaOverride(*viaOverride, ctx, node)
 
-	body, err := ioutil.ReadAll(bufio.NewReader(os.Stdin))
-	if err != nil {
-		log.Fatalln("Can not read body from stdin:", err)
-	}
-
 	if err = ctx.TxExec(
 		node,
 		nice,
 		replyNice,
 		flag.Args()[1],
 		flag.Args()[2:],
-		body,
+		bufio.NewReader(os.Stdin),
 		int64(*minSize)*1024,
 	); err != nil {
 		log.Fatalln(err)
diff --git a/src/cmd/nncp-pkt/main.go b/src/cmd/nncp-pkt/main.go
index 82da1d3..1808e3a 100644
--- a/src/cmd/nncp-pkt/main.go
+++ b/src/cmd/nncp-pkt/main.go
@@ -21,7 +21,6 @@ package main
 import (
 	"bufio"
 	"bytes"
-	"compress/zlib"
 	"flag"
 	"fmt"
 	"io"
@@ -29,6 +28,7 @@ import (
 	"os"
 
 	"github.com/davecgh/go-xdr/xdr2"
+	"github.com/klauspost/compress/zstd"
 	"go.cypherpunks.ru/nncp/v5"
 )
 
@@ -44,7 +44,7 @@ func main() {
 	var (
 		overheads  = flag.Bool("overheads", false, "Print packet overheads")
 		dump       = flag.Bool("dump", false, "Write decrypted/parsed payload to stdout")
-		decompress = flag.Bool("decompress", false, "Try to zlib decompress dumped data")
+		decompress = flag.Bool("decompress", false, "Try to zstd decompress dumped data")
 		cfgPath    = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file")
 		version    = flag.Bool("version", false, "Print version information")
 		warranty   = flag.Bool("warranty", false, "Print warranty information")
@@ -77,13 +77,13 @@ func main() {
 	}
 	var pkt nncp.Pkt
 	_, err = xdr.Unmarshal(bytes.NewReader(beginning), &pkt)
-	if err == nil && pkt.Magic == nncp.MagicNNCPPv2 {
+	if err == nil && pkt.Magic == nncp.MagicNNCPPv3 {
 		if *dump {
 			bufW := bufio.NewWriter(os.Stdout)
 			var r io.Reader
 			r = bufio.NewReader(os.Stdin)
 			if *decompress {
-				decompressor, err := zlib.NewReader(r)
+				decompressor, err := zstd.NewReader(r)
 				if err != nil {
 					log.Fatalln(err)
 				}
diff --git a/src/go.mod b/src/go.mod
index 0ce4d58..b15af41 100644
--- a/src/go.mod
+++ b/src/go.mod
@@ -4,8 +4,10 @@ require (
 	github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892
 	github.com/dustin/go-humanize v1.0.0
 	github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6
+	github.com/google/go-cmp v0.3.1 // indirect
 	github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
 	github.com/hjson/hjson-go v3.0.1+incompatible
+	github.com/klauspost/compress v1.9.2
 	github.com/kr/pretty v0.1.0 // indirect
 	go.cypherpunks.ru/balloon v1.1.0
 	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
diff --git a/src/go.sum b/src/go.sum
index 8fa6343..7c2b3c9 100644
--- a/src/go.sum
+++ b/src/go.sum
@@ -4,10 +4,14 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY=
 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA=
 github.com/hjson/hjson-go v3.0.1+incompatible h1:JwOXblcMiBbiWue7iPkoFK9oXSnW8n+qXh/0Fio6TCo=
 github.com/hjson/hjson-go v3.0.1+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
+github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY=
+github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
diff --git a/src/pkt.go b/src/pkt.go
index cb75c23..2fab844 100644
--- a/src/pkt.go
+++ b/src/pkt.go
@@ -51,7 +51,7 @@ const (
 )
 
 var (
-	MagicNNCPPv2 [8]byte = [8]byte{'N', 'N', 'C', 'P', 'P', 0, 0, 2}
+	MagicNNCPPv3 [8]byte = [8]byte{'N', 'N', 'C', 'P', 'P', 0, 0, 3}
 	MagicNNCPEv4 [8]byte = [8]byte{'N', 'N', 'C', 'P', 'E', 0, 0, 4}
 	BadMagic     error   = errors.New("Unknown magic number")
 	BadPktType   error   = errors.New("Unknown packet type")
@@ -123,7 +123,7 @@ func NewPkt(typ PktType, nice uint8, path []byte) (*Pkt, error) {
 		return nil, errors.New("Too long path")
 	}
 	pkt := Pkt{
-		Magic:   MagicNNCPPv2,
+		Magic:   MagicNNCPPv3,
 		Type:    typ,
 		Nice:    nice,
 		PathLen: uint8(len(path)),
diff --git a/src/toss.go b/src/toss.go
index 2c4726f..0ce446a 100644
--- a/src/toss.go
+++ b/src/toss.go
@@ -20,7 +20,6 @@ package nncp
 import (
 	"bufio"
 	"bytes"
-	"compress/zlib"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -35,6 +34,7 @@ import (
 
 	"github.com/davecgh/go-xdr/xdr2"
 	"github.com/dustin/go-humanize"
+	"github.com/klauspost/compress/zstd"
 	"golang.org/x/crypto/blake2b"
 	"golang.org/x/crypto/poly1305"
 )
@@ -58,6 +58,11 @@ func (ctx *Ctx) Toss(
 	dryRun, doSeen, noFile, noFreq, noExec, noTrns bool,
 ) bool {
 	isBad := false
+	decompressor, err := zstd.NewReader(nil)
+	if err != nil {
+		panic(err)
+	}
+	defer decompressor.Close()
 	for job := range ctx.Jobs(nodeId, TRx) {
 		pktName := filepath.Base(job.Fd.Name())
 		sds := SDS{"node": job.PktEnc.Sender, "pkt": pktName}
@@ -117,10 +122,6 @@ func (ctx *Ctx) Toss(
 				"type": "exec",
 				"dst":  strings.Join(append([]string{handle}, args...), " "),
 			})
-			decompressor, err := zlib.NewReader(pipeR)
-			if err != nil {
-				log.Fatalln(err)
-			}
 			sender := ctx.Neigh[*job.PktEnc.Sender]
 			cmdline, exists := sender.Exec[handle]
 			if !exists || len(cmdline) == 0 {
@@ -128,6 +129,9 @@ func (ctx *Ctx) Toss(
 				isBad = true
 				goto Closing
 			}
+			if err = decompressor.Reset(pipeR); err != nil {
+				log.Fatalln(err)
+			}
 			if !dryRun {
 				cmd := exec.Command(
 					cmdline[0],
diff --git a/src/toss_test.go b/src/toss_test.go
index a96f78b..4ccf4ea 100644
--- a/src/toss_test.go
+++ b/src/toss_test.go
@@ -26,6 +26,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
 	"testing/quick"
 
@@ -96,7 +97,7 @@ func TestTossExec(t *testing.T) {
 				replyNice,
 				handle,
 				[]string{"arg0", "arg1"},
-				[]byte("BODY\n"),
+				strings.NewReader("BODY\n"),
 				1<<15,
 			); err != nil {
 				t.Error(err)
@@ -441,7 +442,7 @@ func TestTossTrns(t *testing.T) {
 		os.MkdirAll(txPath, os.FileMode(0700))
 		for _, data := range datum {
 			pktTrans := Pkt{
-				Magic:   MagicNNCPPv2,
+				Magic:   MagicNNCPPv3,
 				Type:    PktTypeTrns,
 				PathLen: blake2b.Size256,
 				Path:    new([MaxPathSize]byte),
diff --git a/src/tx.go b/src/tx.go
index cb8153f..a509207 100644
--- a/src/tx.go
+++ b/src/tx.go
@@ -20,7 +20,6 @@ package nncp
 import (
 	"bufio"
 	"bytes"
-	"compress/zlib"
 	"crypto/rand"
 	"errors"
 	"hash"
@@ -32,6 +31,7 @@ import (
 	"strings"
 
 	"github.com/davecgh/go-xdr/xdr2"
+	"github.com/klauspost/compress/zstd"
 	"golang.org/x/crypto/blake2b"
 	"golang.org/x/crypto/chacha20poly1305"
 )
@@ -381,7 +381,7 @@ func (ctx *Ctx) TxExec(
 	nice, replyNice uint8,
 	handle string,
 	args []string,
-	body []byte,
+	in io.Reader,
 	minSize int64,
 ) error {
 	path := make([][]byte, 0, 1+len(args))
@@ -394,14 +394,18 @@ func (ctx *Ctx) TxExec(
 		return err
 	}
 	var compressed bytes.Buffer
-	compressor, err := zlib.NewWriterLevel(&compressed, zlib.BestCompression)
+	compressor, err := zstd.NewWriter(
+		&compressed,
+		zstd.WithEncoderLevel(zstd.SpeedDefault),
+	)
 	if err != nil {
 		return err
 	}
-	if _, err = io.Copy(compressor, bytes.NewReader(body)); err != nil {
+	_, err = io.Copy(compressor, in)
+	compressor.Close()
+	if err != nil {
 		return err
 	}
-	compressor.Close()
 	size := int64(compressed.Len())
 	_, err = ctx.Tx(node, pkt, nice, size, minSize, &compressed)
 	sds := SDS{