From: Sergey Matveev Date: Sun, 17 Jan 2021 18:03:15 +0000 (+0300) Subject: Merge branch 'develop' X-Git-Tag: v5.6.0^0 X-Git-Url: http://www.git.cypherpunks.ru/?p=nncp.git;a=commitdiff_plain;h=bb3000c2b40873c8abd5fc9ab663afe2e01005fb;hp=c733b75e61964fa2ce15ad45d0ee987fc5f2afb0 Merge branch 'develop' --- diff --git a/.gitignore b/.gitignore index b7ee15f..fd85e30 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1 @@ -gopath -module-name VERSION diff --git a/bin/default.do b/bin/default.do index bda5482..352b014 100644 --- a/bin/default.do +++ b/bin/default.do @@ -1,12 +1,10 @@ -cd .. -redo-ifchange config gopath module-name -. ./config -. ./gopath -mod=`cat module-name` -redo-ifchange src/*.go src/cmd/$1/*.go +cd ../src +redo-ifchange ../config *.go cmd/$1/*.go +. ../config +GO=${GO:-go} +mod=`$GO list -m` GO_LDFLAGS="$GO_LDFLAGS -X $mod.DefaultCfgPath=$CFGPATH" GO_LDFLAGS="$GO_LDFLAGS -X $mod.DefaultSendmailPath=$SENDMAIL" GO_LDFLAGS="$GO_LDFLAGS -X $mod.DefaultSpoolPath=$SPOOLPATH" GO_LDFLAGS="$GO_LDFLAGS -X $mod.DefaultLogPath=$LOGPATH" -cd src -GOPATH=$GOPATH ${GO:-go} build -o ../bin/$3 -ldflags "$GO_LDFLAGS" $mod/cmd/$1 +$GO build -o ../bin/$3 -ldflags "$GO_LDFLAGS" ./cmd/$1 diff --git a/clean.do b/clean.do index 8ccd8cf..ebefc3e 100644 --- a/clean.do +++ b/clean.do @@ -1,2 +1,2 @@ redo bin/clean -rm -f gopath module-name VERSION +rm -f VERSION diff --git a/doc/call.texi b/doc/call.texi index 447b764..ac24b2a 100644 --- a/doc/call.texi +++ b/doc/call.texi @@ -11,6 +11,9 @@ calls: [ cron: "*/1 * * * MON-FRI" onlinedeadline: 3600 nice: PRIORITY+10 + + autotoss: true + autotoss-doseen: true }, { cron: "30 * * * SAT,SUN" @@ -185,4 +188,9 @@ configuration option when calling. Optional. Override @ref{CfgMaxOnlineTime, @emph{maxonlinetime}} configuration option when calling. +@item autotoss, -doseen, -nofile, -nofreq, -noexec, -notrns +Optionally enable auto tossing: run tosser on node's spool every second +during the call. You can control either are @file{.seen} files must be +created, or skip any kind of packet processing. + @end table diff --git a/doc/cfg.texi b/doc/cfg.texi index f918385..3e0b882 100644 --- a/doc/cfg.texi +++ b/doc/cfg.texi @@ -142,7 +142,7 @@ Dictionary consisting of handles and corresponding command line arguments. In example above there are @command{sendmail} handles, @command{warcer}, @command{wgeter} and @command{flag} one. Remote node can queue some handle execution with providing additional command line -arguments and the body fed to command's stdin. +arguments and the body fed to command's @code{stdin}. @verb{|sendmail: ["/usr/sbin/sendmail", "-t"]|} handle, when called by @verb{|echo hello world | nncp-exec OURNODE sendmail ARG0 ARG1 ARG2|} @@ -190,9 +190,9 @@ human-readable name of the address. For direct TCP connections use @verb{|host:port|} format, pointing to @ref{nncp-daemon}'s listening instance. Also you can pipe connection through the external command using @verb{#|some command#} format. @code{/bin/sh -c "some command"} -will start and its stdin/stdout used as a connection. May be omitted if -either no direct connection exists, or @ref{nncp-call} is used with -forced address specifying. +will start and its @code{stdin}/@code{stdout} used as a connection. May +be omitted if either no direct connection exists, or @ref{nncp-call} is +used with forced address specifying. @anchor{CfgXxRate} @item rxrate/txrate diff --git a/doc/cmds.texi b/doc/cmds.texi index 575807a..ae7bd52 100644 --- a/doc/cmds.texi +++ b/doc/cmds.texi @@ -53,10 +53,10 @@ $ nncp-bundle [options] -rx [-check] [-dryrun] [NODE ...] < ... With @option{-tx} option, this command creates @ref{Bundles, bundle} of @ref{Encrypted, encrypted packets} from the spool directory and writes -it to stdout. +it to @code{stdout}. -With @option{-rx} option, this command takes bundle from stdin and -copies all found packets for our node to the spool directory. Pay +With @option{-rx} option, this command takes bundle from @code{stdin} +and copies all found packets for our node to the spool directory. Pay attention that @strong{no} integrity checking is done by default. Modern tape drives could easily provide too much throughput your CPU won't be able to verify on the fly. So if you won't @ref{nncp-toss, toss} @@ -73,7 +73,7 @@ When packets are sent through the stream, they are still kept in the spool directory, because there is no assurance that they are transferred to the media (media (CD-ROM, tape drive, raw hard drive) can end). If you want to forcefully delete them (after they are successfully flushed -to stdout) anyway, use @option{-delete} option. +to @code{stdout}) anyway, use @option{-delete} option. But you can verify produced stream after, by digesting it by yourself with @option{-rx} and @option{-delete} options -- in that mode, stream @@ -139,8 +139,8 @@ file is renamed from @file{.part} one and when you rerun @command{nncp-call} again, remote node will receive completion notification. -@option{-autotoss} options runs tosser on node's spool after call -is finished. All @option{-autotoss-*} options is the same as in +@option{-autotoss} option runs tosser on node's spool every second +during the call. All @option{-autotoss-*} options is the same as in @ref{nncp-toss} command. @node nncp-caller @@ -219,8 +219,8 @@ $ nncp-cfgnew [options] [-nocomments] > new.hjson @end example Generate new node configuration: private keys, example configuration -file and print it to stdout. You must use this command when you setup -the new node. @option{-nocomments} will create configuration file +file and print it to @code{stdout}. You must use this command when you +setup the new node. @option{-nocomments} will create configuration file without descriptive huge comments -- useful for advanced users. Pay attention that private keys generation consumes an entropy from your @@ -255,16 +255,17 @@ can handle. @option{-bind} option specifies @option{addr:port} it must bind to and listen. It could be run as @command{inetd} service, by specifying -@option{-inetd} option. Pay attention that because it uses stdin/stdout, -it can not effectively work with IO timeouts and connection closing can -propagate up to 5 minutes in practice. Example inetd-entry: +@option{-inetd} option. Pay attention that because it uses +@code{stdin}/@code{stdout}, it can not effectively work with IO timeouts +and connection closing can propagate up to 5 minutes in practice. +Example inetd-entry: @verbatim uucp stream tcp6 nowait nncpuser /usr/local/bin/nncp-daemon nncp-daemon -quiet -inetd @end verbatim -@option{-autotoss} options runs tosser on node's spool after call -is finished. All @option{-autotoss-*} options is the same as in +@option{-autotoss} option runs tosser on node's spool every second +during the call. All @option{-autotoss-*} options is the same as in @ref{nncp-toss} command. @node nncp-exec @@ -275,11 +276,15 @@ $ nncp-exec [options] [-use-tmp] [-nocompress] NODE HANDLE [ARG0 ARG1 ...] @end example Send execution command to @option{NODE} for specified @option{HANDLE}. -Body is read from stdin (either into memory, or into encrypted temporary -file if @option{-use-tmp} is specified) and compressed (unless +Body is read from @code{stdin} into memory and compressed (unless @option{-nocompress} is specified). After receiving, remote side will execute specified @ref{CfgExec, handle} command with @option{ARG*} -appended and decompressed body fed to command's stdin. +appended and decompressed body fed to command's @code{stdin}. + +If @option{-use-tmp} option is specified, then @code{stdin} data is read +into temporary file first, requiring twice more disk space, but no +memory requirements. @ref{StdinTmpFile, Same temporary file} rules +applies as with @ref{nncp-file, nncp-file -} command. For example, if remote side has following configuration file for your node: @@ -325,16 +330,19 @@ This command queues file in @ref{Spool, spool} directory immediately (through the temporary file of course) -- so pay attention that sending 2 GiB file will create 2 GiB outbound encrypted packet. +@anchor{StdinTmpFile} If @file{SRC} equals to @file{-}, then create an encrypted temporary -file and copy everything taken from stdin to it and use for outbound +file and copy everything taken from @code{stdin} to it and use for outbound packet creation. Pay attention that if you want to send 1 GiB of data -taken from stdin, then you have to have more than 2 GiB of disk space +taken from @code{stdin}, then you have to have more than 2 GiB of disk space for that temporary file and resulting encrypted packet. You can control -where temporary file will be stored using @env{TMPDIR} environment +temporary file location directory with @env{TMPDIR} environment variable. Encryption is performed in AEAD mode with @url{https://cr.yp.to/chacha.html, ChaCha20}-@url{https://en.wikipedia.org/wiki/Poly1305, Poly1305} algorithms. Data is splitted on 128 KiB blocks. Each block is encrypted -with increasing nonce counter. +with increasing nonce counter. File is deletes immediately after +creation, so even if program crashes -- disk space will be reclaimed, no +need in cleaning it up later. If @file{SRC} points to directory, then @url{https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_01, pax archive} @@ -402,7 +410,7 @@ Sender: 2WHBV3TPZHDOZGUJEH563ZEK7M33J4UESRFO4PDKWD5KZNPROABQ @end example If you specify @option{-dump} option and provide an @ref{Encrypted, -encrypted} packet, then it will verify and decrypt it to stdout. +encrypted} packet, then it will verify and decrypt it to @code{stdout}. Encrypted packets contain @ref{Plain, plain} ones, that also can be fed to @command{nncp-pkt}: @@ -461,10 +469,10 @@ If @option{-keep} option is specified, then no @file{.nncp.meta}/@file{.nncp.chunkXXX} files are deleted during reassembly process. -@option{-stdout} option outputs reassembled file to stdout, instead of -saving to temporary file with renaming after. This could be useful for -reassembling on separate filesystem to lower fragmentation effect, -and/or separate storage device for higher performance. +@option{-stdout} option outputs reassembled file to @code{stdout}, +instead of saving to temporary file with renaming after. This could be +useful for reassembling on separate filesystem to lower fragmentation +effect, and/or separate storage device for higher performance. @option{-dump} option prints meta-file contents in human-friendly form. It is useful mainly for debugging purposes. For example: diff --git a/doc/comparison.texi b/doc/comparison.texi index 1cb3664..7bf4e53 100644 --- a/doc/comparison.texi +++ b/doc/comparison.texi @@ -120,7 +120,7 @@ comparable commands of UUCP and NNCP, just for the interest: @item Receive connection (pipe, daemon, etc) @tab @command{uucico} (@option{-l} or similar) @tab @command{nncp-daemon} -@item Request remote execution, stdin piped in +@item Request remote execution, @code{stdin} piped in @tab @command{uux} @tab @command{nncp-exec} @item Copy file to remote machine diff --git a/doc/download.texi b/doc/download.texi index 7c4cd67..e372436 100644 --- a/doc/download.texi +++ b/doc/download.texi @@ -25,6 +25,10 @@ Tarballs include all necessary required libraries: @multitable {XXXXX} {XXXX-XX-XX} {XXXX KiB} {link sign} {xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx} @headitem Version @tab Date @tab Size @tab Tarball @tab SHA256 checksum +@item @ref{Release 5.5.1, 5.5.1} @tab 2021-01-11 @tab 1165 KiB +@tab @url{download/nncp-5.5.1.tar.xz, link} @url{download/nncp-5.5.1.tar.xz.sig, sign} +@tab @code{E7DEED7A D3BA696C F64359C0 DC0A93AD 109950C5 6660D028 5FD7BB57 120C9CF7} + @item @ref{Release 5.5.0, 5.5.0} @tab 2021-01-07 @tab 1161 KiB @tab @url{download/nncp-5.5.0.tar.xz, link} @url{download/nncp-5.5.0.tar.xz.sig, sign} @tab @code{EF0CBEE1 520BE97D A210794C 172BF444 E6F75DB2 84F5BD05 66919193 326AED77} diff --git a/doc/integration.texi b/doc/integration.texi index c9b3a71..3359a3a 100644 --- a/doc/integration.texi +++ b/doc/integration.texi @@ -243,7 +243,7 @@ exec: @{ @} @end example -The -bS option is what tells Exim to receive BSMTP on stdin. +The -bS option is what tells Exim to receive BSMTP on @code{stdin}. Now, you need to tell Exim that nncp is a trusted user (able to set From headers arbitrarily). Assuming you are running NNCP as the nncp user, diff --git a/doc/news.ru.texi b/doc/news.ru.texi index c4ce41d..97c108c 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -1,6 +1,28 @@ @node Новости @section Новости +@node Релиз 5.6.0 +@subsection Релиз 5.6.0 +@itemize + +@item +@option{-autotoss*} опции запускают tosser не после завершения вызова, а +во время него ежесекундно. + +@item +В @option{calls} секции конфигурации появились опции +@option{autotoss}, @option{autotoss-doseen}, +@option{autotoss-nofile}, @option{autotoss-nofreq}, +@option{autotoss-noexec}, @option{autotoss-notrns}. +Вы можете настраивать опции автоматического tosser для каждого вызова. + +@item +Использовать vendoring вместо переопределения @env{GOPATH} во время +установки tarball, так как текущая минимальная версия Go это 1.12, +поддерживающая модули. + +@end itemize + @node Релиз 5.5.1 @subsection Релиз 5.5.1 @itemize @@ -446,9 +468,10 @@ HKDF-BLAKE2b-256 KDF алгоритм заменён на BLAKE2Xb XOF. Ещё @item @command{nncp-bundle} команда может создавать потоки зашифрованных -пакетов или потреблять их. Это полезно когда речь идёт о stdin/stdout -методах передачи (например запись на CD-ROM без создания промежуточного -подготовленного ISO образа или работа с ленточными накопителями). +пакетов или потреблять их. Это полезно когда речь идёт о +@code{stdin}/@code{stdout} методах передачи (например запись на CD-ROM +без создания промежуточного подготовленного ISO образа или работа с +ленточными накопителями). @item @command{nncp-toss} команда может создавать @file{.seen} файлы, diff --git a/doc/news.texi b/doc/news.texi index a24704f..83b6f96 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -3,6 +3,28 @@ See also this page @ref{Новости, on russian}. +@node Release 5.6.0 +@section Release 5.6.0 +@itemize + +@item +@option{-autotoss*} option runs tosser not after the call, but every +second while it is active. + +@item +@option{autotoss}, @option{autotoss-doseen}, +@option{autotoss-nofile}, @option{autotoss-nofreq}, +@option{autotoss-noexec}, @option{autotoss-notrns} options available in +@option{calls} configuration section. You can configure per-call +automatic tosser options. + +@item +Use vendoring, instead of @env{GOPATH} overriding during tarball +installation, because current minimal Go's version is 1.12 and it +supports modules. + +@end itemize + @node Release 5.5.1 @section Release 5.5.1 @itemize @@ -435,9 +457,10 @@ are not supported. @item @command{nncp-bundle} command can either create stream of encrypted -packets, or digest it. It is useful when dealing with stdin/stdout based -transmission methods (like writing to CD-ROM without intermediate -prepared ISO image and working with tape drives). +packets, or digest it. It is useful when dealing with +@code{stdin}/@code{stdout} based transmission methods (like writing to +CD-ROM without intermediate prepared ISO image and working with tape +drives). @item @command{nncp-toss} is able to create @file{.seen} files preventing @@ -499,8 +522,8 @@ option for unchunked transfer was not in KiBs, but in bytes. @itemize @item -Ability to feed @command{nncp-file} from stdin, that uses an encrypted -temporary file for that. +Ability to feed @command{nncp-file} from @code{stdin}, that uses an +encrypted temporary file for that. @item Chunked files transmission appeared with corresponding diff --git a/gopath.do b/gopath.do deleted file mode 100644 index 141e16d..0000000 --- a/gopath.do +++ /dev/null @@ -1 +0,0 @@ -echo GOPATH=${GOPATH:-`pwd`} diff --git a/makedist.sh b/makedist.sh index e62491f..8537c9e 100755 --- a/makedist.sh +++ b/makedist.sh @@ -8,109 +8,26 @@ release=$1 git clone . $tmp/nncp-$release cd $tmp/nncp-$release git checkout v$release -redo module-name VERSION -rm -r .redo -mod_name=`cat module-name` -rm -fr .git - -mv src src.orig -mkdir -p src/$mod_name -mv src.orig/* src/$mod_name -rmdir src.orig - -mods=" -github.com/davecgh/go-xdr -github.com/dustin/go-humanize -github.com/flynn/noise -github.com/gorhill/cronexpr -github.com/hjson/hjson-go -github.com/klauspost/compress -go.cypherpunks.ru/balloon -golang.org/x/crypto -golang.org/x/net -golang.org/x/sys -golang.org/x/term -" -for mod in $mods; do - mod_path=$(sed -n "s# // indirect## ; s#^ \($mod\) \(.*\)\$#\1@\2#p" src/$mod_name/go.mod) - [ -n "$mod_path" ] - mkdir -p src/$mod - ( cd $GOPATH/pkg/mod/$mod_path ; tar cf - --exclude ".git*" * ) | tar xfC - src/$mod - chmod -R +w src/$mod -done - -cat > $tmp/includes < $tmp/includes < $tmp/excludes < gopath.do - -perl -p -i -e "s#src/#src/$mod_name/#g" bin/default.do cat > doc/download.texi <.` ) var ( - Version string = "5.5.1" + Version string = "5.6.0" Base32Codec *base32.Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) ) diff --git a/src/toss.go b/src/toss.go index f70ffd3..75e01c4 100644 --- a/src/toss.go +++ b/src/toss.go @@ -33,6 +33,7 @@ import ( "path/filepath" "strconv" "strings" + "time" xdr "github.com/davecgh/go-xdr/xdr2" "github.com/dustin/go-humanize" @@ -99,11 +100,9 @@ func (ctx *Ctx) Toss( ) job.Fd.Close() // #nosec G104 if err != nil { - ctx.LogE("rx", sds, err, "decryption") return pipeW.CloseWithError(err) } if err = pipeWB.Flush(); err != nil { - ctx.LogE("rx", sds, err, "decryption flush") return pipeW.CloseWithError(err) } return pipeW.Close() @@ -156,7 +155,7 @@ func (ctx *Ctx) Toss( if !dryRun { cmd := exec.Command( cmdline[0], - append(cmdline[1:len(cmdline)], args...)..., + append(cmdline[1:], args...)..., ) cmd.Env = append( cmd.Env, @@ -183,7 +182,7 @@ func (ctx *Ctx) Toss( if exists { cmd := exec.Command( sendmail[0], - append(sendmail[1:len(sendmail)], notify.To)..., + append(sendmail[1:], notify.To)..., ) cmd.Stdin = newNotification(notify, fmt.Sprintf( "Exec from %s: %s", sender.Name, argsStr, @@ -304,7 +303,7 @@ func (ctx *Ctx) Toss( if len(sendmail) > 0 && ctx.NotifyFile != nil { cmd := exec.Command( sendmail[0], - append(sendmail[1:len(sendmail)], ctx.NotifyFile.To)..., + append(sendmail[1:], ctx.NotifyFile.To)..., ) cmd.Stdin = newNotification(ctx.NotifyFile, fmt.Sprintf( "File from %s: %s (%s)", @@ -373,7 +372,7 @@ func (ctx *Ctx) Toss( if len(sendmail) > 0 && ctx.NotifyFreq != nil { cmd := exec.Command( sendmail[0], - append(sendmail[1:len(sendmail)], ctx.NotifyFreq.To)..., + append(sendmail[1:], ctx.NotifyFreq.To)..., ) cmd.Stdin = newNotification(ctx.NotifyFreq, fmt.Sprintf( "Freq from %s: %s", sender.Name, src, @@ -426,3 +425,26 @@ func (ctx *Ctx) Toss( } return isBad } + +func (ctx *Ctx) AutoToss( + nodeId *NodeId, + nice uint8, + doSeen, noFile, noFreq, noExec, noTrns bool, +) (chan struct{}, chan bool) { + finish := make(chan struct{}) + badCode := make(chan bool) + go func() { + bad := false + for { + select { + case <-finish: + badCode <- bad + break + default: + } + time.Sleep(time.Second) + bad = !ctx.Toss(nodeId, nice, false, doSeen, noFile, noFreq, noExec, noTrns) + } + }() + return finish, badCode +} diff --git a/test.do b/test.do index feb7b5f..13b228a 100644 --- a/test.do +++ b/test.do @@ -1,7 +1,6 @@ -redo-ifchange config gopath module-name +redo-ifchange config exec >&2 . ./config -. ./gopath -mod=`cat module-name` cd src -GOPATH=$GOPATH ${GO:-go} test -failfast $mod/... +GO=${GO:-go} +$GO test -failfast ./...