From: Sergey Matveev Date: Thu, 7 Jan 2021 14:29:58 +0000 (+0300) Subject: Merge branch 'develop' X-Git-Tag: v5.5.0^0 X-Git-Url: http://www.git.cypherpunks.ru/?a=commitdiff_plain;h=1c773d7a2acd7fef4b7b1567b59e1601a79d55fe;hp=8182577d85f1554bab4a97d6e99e38c2cd13a0d4;p=nncp.git Merge branch 'develop' --- diff --git a/README b/README index 1650f2a..e8bb290 100644 --- a/README +++ b/README @@ -25,6 +25,6 @@ patches to nncp-devel mailing list: https://lists.cypherpunks.ru/pipermail/nncp-devel/ Development Git source code repository currently is located here: -http://git.cypherpunks.ru/cgit.cgi/nncp.git/ +http://www.git.cypherpunks.ru/?p=nncp.git;a=summary For further information please read either doc/nncp.info or doc/nncp.texi. diff --git a/README.RU b/README.RU index d197209..9642305 100644 --- a/README.RU +++ b/README.RU @@ -30,7 +30,7 @@ NNCP это свободное программное обеспечением: https://lists.cypherpunks.ru/pipermail/nncp-devel/ Исходный код для разработчика находится в Git репозитории: -http://git.cypherpunks.ru/cgit.cgi/nncp.git/ +http://www.git.cypherpunks.ru/?p=nncp.git;a=summary Для дополнительной информации пожалуйста читайте или doc/nncp.info или doc/nncp.texi. diff --git a/clean.do b/clean.do index 9c9f7cd..8ccd8cf 100644 --- a/clean.do +++ b/clean.do @@ -1,2 +1,2 @@ redo bin/clean -rm gopath module-name VERSION +rm -f gopath module-name VERSION diff --git a/doc/admin.texi b/doc/admin.texi new file mode 100644 index 0000000..02c1579 --- /dev/null +++ b/doc/admin.texi @@ -0,0 +1,99 @@ +@node Administration +@unnumbered Administration + +NNCP uses following files/directories you should be aware of: + +@itemize + +@item + @ref{Configuration} file, for example @file{/usr/local/etc/nncp.hjson}. + Should not be globally readable, because it contains private keys. + +@item + @ref{Spool} directory with inbound/outbound encrypted packets, for + example @file{/var/spool/nncp}. + + It also contains temporary files (in @file{tmp/} directory), + possibly left alone, if some of the commands failed and prematurely + exited and that should be cleaned from time to time. + + Partly transferred files are stored with @file{.part} suffix. And + possibly they also require cleanup if they are completely lost. + + Also it can contain @file{.seen} files, that should be cleaned too + from time to time. + + All of that cleaning tasks can be done with @ref{nncp-rm} utility. + + @anchor{Shared spool} + If you want to share single spool directory with multiple grouped + Unix users, then you can @command{setgid} it and assure that umask + is group friendly. For convenience you can set @option{umask} + globally for invoked NNCP commands in the configuration file. For + example: + +@example +$ chgrp nncp /usr/local/etc/nncp.hjson /var/spool/nncp +$ chmod g+r /usr/local/etc/nncp.hjson +$ chmod g+rwxs /var/spool/nncp +$ echo 'umask: "007"' >> /usr/local/etc/nncp.hjson +@end example + +@item + @ref{Log} file, for example @file{/var/spool/nncp/log}. It should be + rotated. Choose you own preferable way to do it. + + Example @url{https://www.newsyslog.org/manual.html, newsyslog}'s entry: +@example +/var/spool/nncp/log 644 7 100 * CYN +@end example + +@item + Optional @ref{CfgIncoming, incoming} directories where uploaded + files are stored. Probably you want to run @ref{nncp-reass} from + time to time to reassemble all chunked uploads. Example crontab + entry: + +@example +*/1 * * * * nncp-reass -all -noprogress +@end example + +@item + Possibly long running @ref{nncp-daemon}, @ref{nncp-caller}, + @ref{nncp-toss} daemons. As all software, they can fail and you + should place them under some supervisor control. + + For example you can use @url{http://cr.yp.to/daemontools.html, + daemontools} for that task to run them under probably existing + @verb{|uucp|} user: + +@example +# mkdir -p /var/service/nncp-toss/log +# chmod 755 /var/service/nncp-toss/log /var/service/nncp-toss +# cd /var/service/nncp-toss + +# cat > run_ <&1 +exec setuidgid uucp /usr/local/bin/nncp-toss -cycle 10 +EOF +# chmod 755 run_ + +# cat > log/run_ <> /usr/local/etc/nncp.hjson -@end example diff --git a/doc/cmds.texi b/doc/cmds.texi index 9fff9e4..575807a 100644 --- a/doc/cmds.texi +++ b/doc/cmds.texi @@ -101,6 +101,7 @@ $ nncp-call [options] [-pkts PKT,PKT,...] [-rxrate INT] [-txrate INT] + [-autotoss*] NODE[:ADDR] [FORCEADDR] @end example @@ -138,6 +139,10 @@ file is renamed from @file{.part} one and when you rerun @command{nncp-call} again, remote node will receive completion notification. +@option{-autotoss} options runs tosser on node's spool after call +is finished. All @option{-autotoss-*} options is the same as in +@ref{nncp-toss} command. + @node nncp-caller @section nncp-caller @@ -152,14 +157,14 @@ Optional number of @option{NODE}s tells to ignore other ones. Otherwise all nodes with specified @emph{calls} configuration field will be called. -Look @ref{nncp-call} for more information. +Look at @ref{nncp-call} for more information. @node nncp-cfgenc @section nncp-cfgenc @example -$ nncp-cfgmin [options] [-s INT] [-t INT] [-p INT] cfg.hjson > cfg.hjson.eblob -$ nncp-cfgmin [options] -d cfg.hjson.eblob > cfg.hjson +$ nncp-cfgenc [options] [-s INT] [-t INT] [-p INT] cfg.hjson > cfg.hjson.eblob +$ nncp-cfgenc [options] -d cfg.hjson.eblob > cfg.hjson @end example This command allows you to encrypt provided @file{cfg.hjson} file with @@ -237,7 +242,7 @@ not used often in practice, if ever. @section nncp-daemon @example -$ nncp-daemon [options] [-maxconn INT] [-bind ADDR] [-inetd] +$ nncp-daemon [options] [-maxconn INT] [-bind ADDR] [-inetd] [-autotoss*] @end example Start listening TCP daemon, wait for incoming connections and run @@ -258,16 +263,22 @@ propagate up to 5 minutes in practice. Example inetd-entry: uucp stream tcp6 nowait nncpuser /usr/local/bin/nncp-daemon nncp-daemon -quiet -inetd @end verbatim +@option{-autotoss} options runs tosser on node's spool after call +is finished. All @option{-autotoss-*} options is the same as in +@ref{nncp-toss} command. + @node nncp-exec @section nncp-exec @example -$ nncp-exec [options] NODE HANDLE [ARG0 ARG1 ...] +$ nncp-exec [options] [-use-tmp] [-nocompress] NODE HANDLE [ARG0 ARG1 ...] @end example Send execution command to @option{NODE} for specified @option{HANDLE}. -Body is read from stdin and compressed. After receiving, remote side -will execute specified @ref{CfgExec, handle} command with @option{ARG*} +Body is read from stdin (either into memory, or into encrypted temporary +file if @option{-use-tmp} is specified) and compressed (unless +@option{-nocompress} is specified). After receiving, remote side will +execute specified @ref{CfgExec, handle} command with @option{ARG*} appended and decompressed body fed to command's stdin. For example, if remote side has following configuration file for your @@ -295,6 +306,9 @@ If @ref{CfgNotify, notification} is enabled on the remote side for exec handles, then it will sent simple letter after successful command execution with its output in message body. +@strong{Pay attention} that packet generated with this command won't be +be chunked. + @node nncp-file @section nncp-file @@ -483,19 +497,30 @@ $ nncp-rm [options] -node NODE -pkt PKT This command is aimed to delete various files from your spool directory: @itemize + @item If @option{-tmp} option is specified, then it will delete all temporary files in @file{spool/tmp} directory. Files may stay in it when commands like @ref{nncp-file} fail for some reason. + @item If @option{-lock} option is specified, then all @file{.lock} files will be deleted in your spool directory. + @item If @option{-pkt} option is specified, then @file{PKT} packet (its Base32 name) will be deleted. This is useful when you see some packet failing to be processed. + @item When either @option{-rx} or @option{-tx} options are specified (maybe both of them), then delete all packets from that given queues. If @option{-part} is given, then delete only @file{.part}ly downloaded ones. If @option{-seen} option is specified, then delete only @file{.seen} files. + +@item @option{-dryrun} option just prints what will be deleted. + +@item You can also select files that only have modification date older +than specified @option{-older} time units (@code{10s} (10 seconds), +@code{5m} (5 minutes), @code{12h} (12 hours), @code{2d} (2 days)). + @end itemize @node nncp-stat diff --git a/doc/comparison.ru.texi b/doc/comparison.ru.texi index 7b02d03..280220c 100644 --- a/doc/comparison.ru.texi +++ b/doc/comparison.ru.texi @@ -12,7 +12,7 @@ @item Простота настройки @tab Средне @tab Сложно @tab Легко @tab Сложно @item Передача почты @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} -@item Передача новостей @tab @strong{Да} @tab @strong{Да} @tab Нет @tab Нет +@item Передача новостей @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab Нет @item Передача файлов @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab Нет @item Разбиение файлов на части @tab Нет @tab @strong{Да} @tab @strong{Да} @tab Нет @item Удалённое исполнение команд @tab @strong{Да} @tab Нет @tab @strong{Да} @tab Нет @@ -51,10 +51,6 @@ NNCP требует редактирование единственного Hjson @ref{Configuration, конфигурационного файла}. -@item Передача новостей - SMTP ничего не знает о новостях, NNTP и тому подобному. NNCP тоже не - знает, потому что на текущий день они уже мало используются. - @item Передача файлов SMTP может передавать файлы только в Base64 кодировке -- это очень не эффективно. @@ -71,11 +67,11 @@ этом в очереди на отправку будут гигабайты файлов. @item Интеграция с SMTP - Почтовые серверы типа @url{http://www.postfix.org/, Postfix} - предоставляют документацию и примеры конфигурации для использования - с UUCP. @url{http://www.exim.org/, Exim} и + Почтовые серверы типа @url{http://www.postfix.org/, Postfix} и + @url{http://www.exim.org/, Exim} предоставляют документацию и + примеры конфигурации для использования с UUCP. @url{http://www.sendmail.com/sm/open_source/, Sendmail} тоже - относительно легко могут быть интегрированы с ним. Для использования + относительно легко может быть интегрирован с ним. Для использования с NNCP, просто замените UUCP команды на аналогичные NNCP. @item Push/poll модель diff --git a/doc/comparison.texi b/doc/comparison.texi index 11fe4a2..1cb3664 100644 --- a/doc/comparison.texi +++ b/doc/comparison.texi @@ -11,7 +11,7 @@ FidoNet} Technology Networks) and @url{https://en.wikipedia.org/wiki/SMTP, SMTP} @item Ease of setup @tab Medium @tab Hard @tab Easy @tab Hard @item Mail transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} -@item News transmission @tab @strong{Yes} @tab @strong{Yes} @tab No @tab No +@item News transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab No @item File transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab No @item Chunked files @tab No @tab @strong{Yes} @tab @strong{Yes} @tab No @item Remote command execution @tab @strong{Yes} @tab No @tab @strong{Yes} @tab No @@ -48,10 +48,6 @@ FidoNet} Technology Networks) and @url{https://en.wikipedia.org/wiki/SMTP, SMTP} NNCP requires editing of single Hjson @ref{Configuration, configuration file}. -@item News transmission - SMTP does not know anything about news, NNTP and so forth. Neither - does NNCP, because they are not used very much nowadays. - @item File transmission SMTP could transfer files only Base64-encoding them -- this is very inefficient. @@ -68,9 +64,9 @@ FidoNet} Technology Networks) and @url{https://en.wikipedia.org/wiki/SMTP, SMTP} gigabytes files are queued in parallel. @item SMTP integration - Mail servers like @url{http://www.postfix.org/, Postfix} offers - documentation and configuration file examples how to use it with - UUCP. @url{http://www.exim.org/, Exim} and + Mail servers like @url{http://www.postfix.org/, Postfix} and + @url{http://www.exim.org/, Exim} offers documentation and + configuration file examples how to use it with UUCP. @url{http://www.sendmail.com/sm/open_source/, Sendmail} could be integrated with UUCP rather easily too. For using NNCP, just replace UUCP commands with NNCP ones. @@ -110,3 +106,37 @@ FidoNet} Technology Networks) and @url{https://en.wikipedia.org/wiki/SMTP, SMTP} requires more manual work to do so. @end table + +Also there is +@url{https://changelog.complete.org/archives/10165-asynchronous-email-exim-over-nncp-or-uucp, copy of} +comparable commands of UUCP and NNCP, just for the interest: + +@multitable @columnfractions 0.5 0.25 0.25 +@headitem Purpose @tab UUCP @tab NNCP + +@item Connect to remote system + @tab @command{uucico -s}, @command{uupoll} + @tab @command{nncp-call}, @command{nncp-caller} +@item Receive connection (pipe, daemon, etc) + @tab @command{uucico} (@option{-l} or similar) + @tab @command{nncp-daemon} +@item Request remote execution, stdin piped in + @tab @command{uux} + @tab @command{nncp-exec} +@item Copy file to remote machine + @tab @command{uucp} + @tab @command{nncp-file} +@item Copy file from remote machine + @tab @command{uucp} + @tab @command{nncp-freq} +@item Process received requests + @tab @command{uuxqt} + @tab @command{nncp-toss} +@item Move outbound requests to dir (for USB stick, airgap, etc) + @tab N/A + @tab @command{nncp-xfer} +@item Create streaming package of outbound requests + @tab N/A + @tab @command{nncp-bundle} + +@end multitable diff --git a/doc/default.plantuml.txt.do b/doc/default.plantuml.txt.do index cb5bf3e..917366a 100644 --- a/doc/default.plantuml.txt.do +++ b/doc/default.plantuml.txt.do @@ -2,4 +2,3 @@ src=${1%.txt} redo-ifchange $src ../config . ../config $PLANTUML -tutxt -pipe < $src - diff --git a/doc/download.texi b/doc/download.texi index faab865..78f802b 100644 --- a/doc/download.texi +++ b/doc/download.texi @@ -24,9 +24,9 @@ Tarballs include all necessary required libraries: @multitable {XXXXX} {XXXX-XX-XX} {XXXX KiB} {link sign} {xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx} @headitem Version @tab Date @tab Size @tab Tarball @tab SHA256 checksum -@item @ref{Release 5.4.0, 5.4.0} @tab 2020-09-27 @tab 1136 KiB -@tab @url{download/nncp-5.4.0.tar.xz, link} @url{download/nncp-5.4.0.tar.xz.sig, sign} -@tab @code{2730F08B EE590B24 0D096EB8 4105BBCB 8054C840 C3610AF9 6C9D50ED E1F0035C} +@item @ref{Release 5.4.1, 5.4.1} @tab 2020-09-28 @tab 1143 KiB +@tab @url{download/nncp-5.4.1.tar.xz, link} @url{download/nncp-5.4.1.tar.xz.sig, sign} +@tab @code{A02D0C9B 51533DF8 115C17E1 02F8C485 9F7B805A 64290CDF 79151BA9 E627FA63} @item @ref{Release 5.3.3, 5.3.3} @tab 2020-01-23 @tab 1116 KiB @tab @url{download/nncp-5.3.3.tar.xz, link} @url{download/nncp-5.3.3.tar.xz.sig, sign} diff --git a/doc/git-bundler.sh b/doc/git-bundler.sh index 70df771..bd7dd15 100755 --- a/doc/git-bundler.sh +++ b/doc/git-bundler.sh @@ -1,13 +1,7 @@ #!/bin/sh -ex tmp=$(mktemp) - -cleanup() -{ - rm -f $tmp -} -trap cleanup HUP PIPE INT QUIT TERM EXIT - +trap "rm -f $tmp" HUP PIPE INT QUIT TERM EXIT read revs cd $HOME/git/$1.git git bundle create $tmp $revs diff --git a/doc/index.texi b/doc/index.texi index 57f1db7..057995c 100644 --- a/doc/index.texi +++ b/doc/index.texi @@ -6,7 +6,7 @@ This manual is for NNCP (Node to Node copy) -- collection of utilities simplifying secure store-and-forward files and mail exchanging. -Copyright @copyright{} 2016-2020 @email{stargrave@@stargrave.org, Sergey Matveev} +Copyright @copyright{} 2016-2021 @email{stargrave@@stargrave.org, Sergey Matveev} @quotation Permission is granted to copy, distribute and/or modify this document @@ -23,6 +23,15 @@ A copy of the license is included in the section entitled "Copying conditions". @include about.texi +There are also articles about its usage outside this website: + +@itemize +@item @url{https://habr.com/en/post/330712/, NNCP: лечение online- и цензуро- зависимости store-and-forward методом} (on russian) +@item @url{https://changelog.complete.org/archives/tag/nncp, Series of descriptive articles} by John Goerzen +@item @url{https://www.youtube.com/watch?v=0i3r63pnyHM, Presentation of NNCP on Internet Freedom Conference 2019} (on russian) + (@url{http://www.stargrave.org/20191207-Internet_Freedom_Conference-NNCP.webm.torrent, torrent}) +@end itemize + @center Interested? @ref{Tarballs, @strong{Download it}}! @menu @@ -36,6 +45,7 @@ A copy of the license is included in the section entitled "Copying conditions". * Call configuration: Call. * Integration:: * Commands:: +* Administration:: * Niceness:: * Chunked files: Chunked. * Bundles:: @@ -59,6 +69,7 @@ A copy of the license is included in the section entitled "Copying conditions". @include call.texi @include integration.texi @include cmds.texi +@include admin.texi @include niceness.texi @include chunked.texi @include bundles.texi diff --git a/doc/integration.texi b/doc/integration.texi index 8879d91..c9b3a71 100644 --- a/doc/integration.texi +++ b/doc/integration.texi @@ -7,6 +7,7 @@ making them store-and-forward friendly. @menu * Index files for freqing: FreqIndex. * Postfix:: +* Exim:: * Web feeds: Feeds. * Web pages: WARCs. * BitTorrent and huge files: BitTorrent. @@ -147,16 +148,121 @@ effective. @end itemize +@node Exim +@section Integration with Exim + +This section is unaltered copy-paste of +@url{https://changelog.complete.org/archives/10165-asynchronous-email-exim-over-nncp-or-uucp, Asynchronous Email: Exim over NNCP (or UUCP)} +article by John Goerzen, with his permission. + +@strong{Sending from Exim to a smarthost} + +One common use for async email is from a satellite system: one that +doesn't receive mail, or have local mailboxes, but just needs to get +email out to the Internet. This is a common situation even for +conventionally-connected systems; in Exim speak, this is a "satellite +system that routes mail via a smarthost". That is, every outbound +message goes to a specific target, which then is responsible for +eventual delivery (over the Internet, LAN, whatever). + +This is fairly simple in Exim. + +We actually have two choices for how to do this: bsmtp or rmail mode. +bsmtp (batch SMTP) is the more modern way, and is essentially a +derivative of SMTP that explicitly can be queued asynchronously. +Basically it's a set of SMTP commands that can be saved in a file. The +alternative is "rmail" (which is just an alias for sendmail these days), +where the data is piped to rmail/sendmail with the recipients given on +the command line. Both can work with Exim and NNCP, but because we're +doing shiny new things, we'll use bsmtp. + +These instructions are loosely based on the +@url{https://people.debian.org/~jdg/bsmtp.html, Using outgoing BSMTP with Exim HOWTO}. +Some of these may assume Debianness in the configuration, but should be +easily enough extrapolated to other configs as well. + +First, configure Exim to use satellite mode with minimal DNS lookups +(assuming that you may not have working DNS anyhow). + +Then, in the Exim primary router section for smarthost +(@file{router/200_exim4-config_primary} in Debian split configurations), +just change @code{transport = remote_smtp_smarthost to transport = nncp}. + +Now, define the NNCP transport. If you are on Debian, you might name this +@file{transports/40_exim4-config_local_nncp}: + +@example +nncp: + debug_print = "T: nncp transport for $local_part@@$domain" + driver = pipe + user = nncp + batch_max = 100 + use_bsmtp + command = /usr/local/nncp/bin/nncp-exec -noprogress -quiet hostname_goes_here rsmtp +.ifdef REMOTE_SMTP_HEADERS_REWRITE + headers_rewrite = REMOTE_SMTP_HEADERS_REWRITE +.endif +.ifdef REMOTE_SMTP_RETURN_PATH + return_path = REMOTE_SMTP_RETURN_PATH +.endif +@end example + +This is pretty straightforward. We pipe to @command{nncp-exec}, run it +as the nncp user. @command{nncp-exec} sends it to a target node and runs +whatever that node has called @command{rsmtp} (the command to receive +bsmtp data). When the target node processes the request, it will run the +configured command and pipe the data in to it. + +@strong{More complicated: Routing to various NNCP nodes} + +Perhaps you would like to be able to send mail directly to various NNCP +nodes. There are a lot of ways to do that. + +Fundamentally, you will need a setup similar to the UUCP example in +@url{https://www.exim.org/exim-html-current/doc/html/spec_html/ch-the_manualroute_router.html, +Exim's manualroute manual}, which lets you define how to reach various +hosts via UUCP/NNCP. Perhaps you have a star topology (every NNCP node +exchanges email with a central hub). In the NNCP world, you have two +choices of how you do this. You could, at the Exim level, make the +central hub the smarthost for all the side nodes, and let it +redistribute mail. That would work, but requires decrypting messages at +the hub to let Exim process. The other alternative is to configure NNCP +to just send to the destinations via the central hub; that takes +advantage of onion routing and doesn't require any Exim processing at +the central hub at all. + +@strong{Receiving mail from NNCP} + +On the receiving side, first you need to configure NNCP to authorize the +execution of a mail program. In the section of your receiving host where +you set the permissions for the client, include something like this: + +@example +exec: @{ + rsmtp: ["/usr/sbin/sendmail", "-bS"] +@} +@end example + +The -bS option is what tells Exim to receive BSMTP on stdin. + +Now, you need to tell Exim that nncp is a trusted user (able to set From +headers arbitrarily). Assuming you are running NNCP as the nncp user, +then add @code{MAIN_TRUSTED_USERS = nncp} to a file such as +@file{/etc/exim4/conf.d/main/01_exim4-config_local-nncp}. That's it! + +Some hosts, of course, both send and receive mail via NNCP and will need +configurations for both. + @node Feeds @section Integration with Web feeds RSS and Atom feeds could be collected using -@url{https://github.com/wking/rss2email, rss2email} program. It -converts all incoming feed entries to email messages. Read about how to -integration @ref{Postfix} with email. @command{rss2email} could be run -in a cron, to collect feeds without any user interaction. Also this -program supports ETags and won't pollute the channel if remote server -supports them too. +@url{https://github.com/wking/rss2email, rss2email} program. It converts +all incoming feed entries to email messages. Read about how to integrate +@ref{Postfix}/@ref{Exim} with email. @command{rss2email} could be run in +a cron, to collect feeds without any user interaction. Also this program +supports ETags and won't pollute the channel if remote server supports +them too. After installing @command{rss2email}, create configuration file: @@ -167,7 +273,7 @@ $ r2e new rss-robot@@address.com and add feeds you want to retrieve: @example -$ r2e add http://git.cypherpunks.ru/cgit.cgi/nncp.git/atom/?h=master +$ r2e add http://www.git.cypherpunks.ru/?p=nncp.git;a=atom @end example and run the process: diff --git a/doc/log.texi b/doc/log.texi index 1b0f43e..9bcff52 100644 --- a/doc/log.texi +++ b/doc/log.texi @@ -26,9 +26,10 @@ I 2017-01-09T08:48:59.264847401Z [call-finish duration="10" node="BYRRQUULEHINPK Is single character log level. As a rule is is either @verb{|I|} (informational message), or @verb{|E|} (error message). @item DATETIME - UTC datetime in RFC 3339 @verb{|2006-01-02T15:04:05.999999999Z|} format. + UTC datetime in @url{https://tools.ietf.org/html/rfc339, RFC 3339} + @verb{|2006-01-02T15:04:05.999999999Z|} format. @item SD - Structured data as in RFC 5424. + Structured data as in @url{https://tools.ietf.org/html/rfc5424, RFC 5424}. @item MSG Arbitrary UTF-8 encoded text data. @end table diff --git a/doc/news.ru.texi b/doc/news.ru.texi index 6f9c98d..d06a15a 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -1,6 +1,31 @@ @node Новости @section Новости +@node Релиз 5.5.0 +@subsection Релиз 5.5.0 +@itemize + +@item +Исправления ошибок в @command{nncp-call(er)}/@command{nncp-daemon}, +@command{nncp-bundle}, @command{nncp-stat} командах. + +@item +У команды @command{nncp-rm} появились @option{-dryrun} и @option{-older} опции. + +@item +У команды @command{nncp-exec} появились @option{-use-tmp} и +@option{-nocompress} опции. Несжатые пакеты не совместимы с предыдущими +версиями NNCP. + +@item +У команд @command{nncp-call}, @command{nncp-caller} и @command{nncp-daemon} +появились @option{-autotoss*} опции для запуска tosser после завершения звонка. + +@item +Обновлены зависимые библиотеки. + +@end itemize + @node Релиз 5.4.1 @subsection Релиз 5.4.1 @itemize diff --git a/doc/news.texi b/doc/news.texi index 37edcc0..ff9a47f 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -3,6 +3,31 @@ See also this page @ref{Новости, on russian}. +@node Release 5.5.0 +@section Release 5.5.0 +@itemize + +@item +Bugfixes in @command{nncp-call(er)}/@command{nncp-daemon}, +@command{nncp-bundle} and @command{nncp-stat}. + +@item +@command{nncp-rm} has @option{-dryrun} and @option{-older} options now. + +@item +@command{nncp-exec} has @option{-use-tmp} and @option{-nocompress} +options now. Uncompressed packets are not compatible with previous NNCP +versions. + +@item +@command{nncp-call}, @command{nncp-caller} and @command{nncp-daemon} commands +have @option{-autotoss*} options for running tosser after call is ended. + +@item +Updated dependencies. + +@end itemize + @node Release 5.4.1 @section Release 5.4.1 @itemize diff --git a/doc/pkt.texi b/doc/pkt.texi index f7bbe56..93841e1 100644 --- a/doc/pkt.texi +++ b/doc/pkt.texi @@ -31,7 +31,7 @@ drive. @verb{|N N C P P 0x00 0x00 0x03|} @item Payload type @tab unsigned integer @tab - 0 (file), 1 (freq), 2 (exec), 3 (transition) + 0 (file), 1 (freq), 2 (exec), 3 (transition), 4 (exec-fat) @item Niceness @tab unsigned integer @tab 1-255, preferred packet @ref{Niceness, niceness} level @@ -50,8 +50,9 @@ drive. Path has fixed size because of hiding its actual length -- it is valuable metadata. Payload is appended to the header -- it is not stored -as XDR field, because most XDR libraries will store all that data in the -memory. +as XDR field, because XDR has no ability to pass more than 4 GiB of +opaque data. Moreover most XDR libraries store fields in the memory in +practice. Depending on the packet's type, payload could store: @@ -60,6 +61,7 @@ Depending on the packet's type, payload could store: @item Destination path for freq @item @url{https://facebook.github.io/zstd/, Zstandard} compressed exec body @item Whole encrypted packet we need to relay on +@item Uncompressed exec body @end itemize Also depending on packet's type, niceness level means: diff --git a/doc/sources.texi b/doc/sources.texi index d1970e6..990639c 100644 --- a/doc/sources.texi +++ b/doc/sources.texi @@ -11,3 +11,5 @@ $ git clone git://git.cypherpunks.ru/nncp.git $ cd nncp $ git checkout develop @end example + +You can also use @url{https://git.cypherpunks.ru/nncp.git}. diff --git a/doc/sp.plantuml b/doc/sp.plantuml index 89440ba..b86ee68 100644 --- a/doc/sp.plantuml +++ b/doc/sp.plantuml @@ -1,34 +1,19 @@ - ┌─────────┐ ┌─────────┐ - │Initiator│ │Responder│ - └────┬────┘ └────┬────┘ - │ │ - │ ╔═════════════╗ │ -══════════╪═════════╣ preparation ╠═════════╪══════════ - │ ╚═════════════╝ │ - │ │ - │ [s] │ - │<────────────────────────────────│ - │ │ - │ │ - │ ╔═════════════╗ │ -══════════╪═════════╣ interactive ╠═════════╪══════════ - │ ╚═════════════╝ │ - │ │ - │[e, es, s, ss], INFO..., HALT... │ - │────────────────────────────────>│ - │ │ - │ [e, ee, se], INFO..., HALT... │ - │<────────────────────────────────│ - │ │ - │ INFO..., FREQ..., DONE... │ - │────────────────────────────────>│ - │ │ - │ INFO..., FREQ..., DONE... │ - │<────────────────────────────────│ - │ │ - │FILE..., INFO..., DONE..., PING │ - │────────────────────────────────>│ - │ │ - │FILE..., INFO..., DONE..., PING │ - │<────────────────────────────────│ - │ │ +@startuml +hide footbox +participant Initiator +participant Responder + +== preparation == + +Initiator <- Responder : [s] + +== interactive == + +Initiator -> Responder : [e, es, s, ss], INFO..., HALT... +Initiator <- Responder : [e, ee, se], INFO..., HALT... +Initiator -> Responder : INFO..., FREQ..., DONE... +Initiator <- Responder : INFO..., FREQ..., DONE... +Initiator -> Responder : FILE..., INFO..., DONE..., PING +Initiator <- Responder : FILE..., INFO..., DONE..., PING + +@enduml diff --git a/doc/thanks.texi b/doc/thanks.texi index c0cb808..f4cee43 100644 --- a/doc/thanks.texi +++ b/doc/thanks.texi @@ -12,6 +12,6 @@ NNCP under Ubuntu GNU/Linux distributions and bug reports. feedback and NixOS package maintenance. @item @url{mailto:jgoerzen@@complete.org, John Goerzen} for his feature -suggestions and Debian package maintenance. +suggestions, bugreports and Debian package maintenance. @end itemize diff --git a/doc/usecases.ru.texi b/doc/usecases.ru.texi index 9e7e178..8f12cd1 100644 --- a/doc/usecases.ru.texi +++ b/doc/usecases.ru.texi @@ -20,10 +20,10 @@ @subsection Доступность почтового сервера время от времени Представьте, что у вас есть собственный @url{http://www.postfix.org/, -Postfix} SMTP сервер подключённый к Интернету. Но вы читаете и пишете -почтовые сообщения на своём ноутбуке, который подключается к нему лишь -время от времени. Как опустошить очередь из ожидающих сообщений когда -ноутбук подключён? +Postfix}/@url{http://www.exim.org/, Exim} SMTP сервер подключённый к +Интернету. Но вы читаете и пишете почтовые сообщения на своём ноутбуке, +который подключается к нему лишь время от времени. Как опустошить +очередь из ожидающих сообщений когда ноутбук подключён? Одна из возможностей это войти на сервер и сделать что-то типа @command{postqueue -f}, но по умолчанию у вас есть только несколько дней @@ -36,15 +36,13 @@ Postfix} SMTP сервер подключённый к Интернету. Но @url{https://ru.wikipedia.org/wiki/KISS_(%D0%BF%D1%80%D0%B8%D0%BD%D1%86%D0%B8%D0%BF), KISS}! -Просто скажите вашим обоим Postfix-ам (на сервере и ноутбуке) отправлять -сообщения через NNCP (@ref{nncp-exec}) на заданный узел. Это делается -аналогично тому как с UUCP, и описано в -@url{http://www.postfix.org/UUCP_README.html, документации Postfix}. - -Читайте @ref{Postfix, здесь} для более подробной информации. Вся почта -будет сохранятся в NNCP @ref{Spool, спуле}, который после обмена данных -и распаковки вызовет локальный @command{sendmail} для доставки почты, -как-будто это произошло на этой же машине. +Просто скажите вашим обоим Postfix/Exim-ам (на сервере и ноутбуке) +отправлять сообщения через NNCP (@ref{nncp-exec}) на заданный узел. +Более подробно читайте для Postfix @ref{Postfix, здесь}, а для Exim +@ref{Exim, здесь}. Вся почта будет сохранятся в NNCP @ref{Spool, спуле}, +который после обмена данных и распаковки вызовет локальный +@command{sendmail} для доставки почты, как-будто это произошло на этой +же машине. @node UsecasePOPRU @subsection Легковесная и быстрая замена POP3/IMAP4 diff --git a/doc/usecases.texi b/doc/usecases.texi index e118da4..0a42c8a 100644 --- a/doc/usecases.texi +++ b/doc/usecases.texi @@ -21,10 +21,11 @@ See also this page @ref{Сценарии, on russian}. @node UsecaseMail @section Occasional connection to mail server -Assume that you have got your own @url{http://www.postfix.org/, Postfix} -SMTP server connected to the Internet. But you read and write emails on -your notebook, that is connected to it just from time to time. How can -you flush buffered mail queues when your notebook is connected? +Assume that you have got your own @url{http://www.postfix.org/, +Postfix}/@url{http://www.exim.org/, Exim} SMTP server connected to the +Internet. But you read and write emails on your notebook, that is +connected to it just from time to time. How can you flush buffered mail +queues when your notebook is connected? One possibility is to log in and run something like @command{postqueue -f}, but by default you have got only several days so and sender will @@ -35,15 +36,13 @@ Another possibility is to use POP3/IMAP4 servers, but this is too overcomplicated and bloated for the simple task. Not an option. @url{https://en.wikipedia.org/wiki/KISS_principle, KISS}! -Just tell both of your Postfixes (on the server and notebook) to drop -email as a mail via NNCP (@ref{nncp-exec}) to specified node. This is -done similarly as with UUCP and as written in -@url{http://www.postfix.org/UUCP_README.html, Postfix documentation}. +Just tell both of your Postfix/Exim (on the server and notebook) to drop +email as a mail via NNCP (@ref{nncp-exec}) to specified node. -Look @ref{Postfix, here} for further information. All mail will be -stored in NNCP @ref{Spool, spool}, that after exchanging and tossing -will call local @command{sendmail} command to deliver them just like -that happened on the same machine. +More information for Postfix is @ref{Postfix, here} and for Exim is +@ref{Exim, here}. All mail will be stored in NNCP @ref{Spool, spool}, +that after exchanging and tossing will call local @command{sendmail} +command to deliver them just like that happened on the same machine. @node UsecasePOP @section Lightweight fast POP3/IMAP4 replacement diff --git a/doc/workflow.texi b/doc/workflow.texi index def2929..8015b16 100644 --- a/doc/workflow.texi +++ b/doc/workflow.texi @@ -13,9 +13,9 @@ neighbours. Add their keys to your configuration file and do any other required configuration about their reachability, permissions of file or freq transmission. @item Use @ref{nncp-file}, @ref{nncp-freq}, @ref{nncp-exec} -(@ref{Postfix, look how} Postfix SMTP server could be configured) -commands to queue file, freq and exec transmissions. Repeat as -many times any time as you wish. +(look @ref{Postfix, how} Postfix and @ref{Exim, how} Exim SMTP servers +could be configured) commands to queue file, freq and exec +transmissions. Repeat as many times any time as you wish. @item Depending on connection methods, either: @itemize @item run @ref{nncp-daemon} to accept remotely initiated connections diff --git a/ports/nncp/files/nncp.newsyslog.conf.sample b/ports/nncp/files/nncp.newsyslog.conf.sample index 46c00fc..0a16524 100644 --- a/ports/nncp/files/nncp.newsyslog.conf.sample +++ b/ports/nncp/files/nncp.newsyslog.conf.sample @@ -1 +1 @@ -/var/spool/nncp/log 644 7 100 * CXN +/var/spool/nncp/log 644 7 100 * CYN diff --git a/ports/nncp/files/pkg-message.in b/ports/nncp/files/pkg-message.in index 3d6ddd9..de4eb3f 100644 --- a/ports/nncp/files/pkg-message.in +++ b/ports/nncp/files/pkg-message.in @@ -11,7 +11,7 @@ - %%PREFIX%%/etc/newsyslog.conf.d/nncp.conf log rotation configuration has been installed. - /var/spool/nncp is the packet spool directory. -- Look in nncp.info how to integrate mail exchanging with Postfix. +- Look in nncp.info how to integrate mail exchanging with Postfix/Exim. - Generate NNCP configuration file using the command: # umask 077 diff --git a/src/call.go b/src/call.go index 65d7d5f..f859167 100644 --- a/src/call.go +++ b/src/call.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cfg.go b/src/cfg.go index df26f1e..a37fbdc 100644 --- a/src/cfg.go +++ b/src/cfg.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/check.go b/src/check.go index 824f0c7..4a1be88 100644 --- a/src/check.go +++ b/src/check.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/chunked.go b/src/chunked.go index 824f49a..573bb9a 100644 --- a/src/chunked.go +++ b/src/chunked.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-bundle/main.go b/src/cmd/nncp-bundle/main.go index 1cfdd45..ae14e89 100644 --- a/src/cmd/nncp-bundle/main.go +++ b/src/cmd/nncp-bundle/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -313,11 +313,12 @@ func main() { continue } } - sds["node"] = nncp.Base32Codec.EncodeToString(pktEnc.Recipient[:]) + sender := nncp.Base32Codec.EncodeToString(pktEnc.Sender[:]) + sds["node"] = sender sds["pkt"] = pktName sds["fullsize"] = entry.Size - selfPath := filepath.Join(ctx.Spool, ctx.SelfId.String(), string(nncp.TRx)) - dstPath := filepath.Join(selfPath, pktName) + dstDirPath := filepath.Join(ctx.Spool, sender, string(nncp.TRx)) + dstPath := filepath.Join(dstDirPath, pktName) if _, err = os.Stat(dstPath); err == nil || !os.IsNotExist(err) { ctx.LogD("nncp-bundle", sds, "Packet already exists") continue @@ -357,7 +358,7 @@ func main() { log.Fatalln("Error during flusing:", err) } if nncp.Base32Codec.EncodeToString(tmp.Hsh.Sum(nil)) == pktName { - if err = tmp.Commit(selfPath); err != nil { + if err = tmp.Commit(dstDirPath); err != nil { log.Fatalln("Error during commiting:", err) } } else { @@ -392,13 +393,13 @@ func main() { if err = tmp.Close(); err != nil { log.Fatalln("Error during closing:", err) } - if err = os.MkdirAll(selfPath, os.FileMode(0777)); err != nil { + if err = os.MkdirAll(dstDirPath, os.FileMode(0777)); err != nil { log.Fatalln("Error during mkdir:", err) } if err = os.Rename(tmp.Name(), dstPath); err != nil { log.Fatalln("Error during renaming:", err) } - if err = nncp.DirSync(selfPath); err != nil { + if err = nncp.DirSync(dstDirPath); err != nil { log.Fatalln("Error during syncing:", err) } } diff --git a/src/cmd/nncp-call/main.go b/src/cmd/nncp-call/main.go index 634c003..52e0fcb 100644 --- a/src/cmd/nncp-call/main.go +++ b/src/cmd/nncp-call/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -58,6 +58,13 @@ func main() { onlineDeadlineSec = flag.Uint("onlinedeadline", 0, "Override onlinedeadline option") maxOnlineTimeSec = flag.Uint("maxonlinetime", 0, "Override maxonlinetime option") + + autotoss = flag.Bool("autotoss", false, "Toss after call is finished") + autotossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autotossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") + autotossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") + autotossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") + autotossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") ) flag.Usage = usage flag.Parse() @@ -153,7 +160,7 @@ func main() { } ctx.Umask() - if !ctx.CallNode( + badCode := !ctx.CallNode( node, addrs, nice, @@ -164,7 +171,20 @@ func main() { maxOnlineTime, *listOnly, onlyPkts, - ) { + ) + if *autotoss { + badCode = ctx.Toss( + node.Id, + nice, + false, + *autotossDoSeen, + *autotossNoFile, + *autotossNoFreq, + *autotossNoExec, + *autotossNoTrns, + ) || badCode + } + if badCode { os.Exit(1) } } diff --git a/src/cmd/nncp-caller/main.go b/src/cmd/nncp-caller/main.go index 81c90c4..421f0fa 100644 --- a/src/cmd/nncp-caller/main.go +++ b/src/cmd/nncp-caller/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -49,6 +49,13 @@ func main() { debug = flag.Bool("debug", false, "Print debug messages") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") + + autotoss = flag.Bool("autotoss", false, "Toss after call is finished") + autotossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autotossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") + autotossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") + autotossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") + autotossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") ) flag.Usage = usage flag.Parse() @@ -145,6 +152,19 @@ func main() { false, nil, ) + if *autotoss { + ctx.Toss( + node.Id, + call.Nice, + false, + *autotossDoSeen, + *autotossNoFile, + *autotossNoFreq, + *autotossNoExec, + *autotossNoTrns, + ) + } + node.Lock() node.Busy = false node.Unlock() diff --git a/src/cmd/nncp-cfgenc/main.go b/src/cmd/nncp-cfgenc/main.go index 036a4c8..7eb6766 100644 --- a/src/cmd/nncp-cfgenc/main.go +++ b/src/cmd/nncp-cfgenc/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-cfgmin/main.go b/src/cmd/nncp-cfgmin/main.go index 20bd14c..25e73d5 100644 --- a/src/cmd/nncp-cfgmin/main.go +++ b/src/cmd/nncp-cfgmin/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-cfgnew/main.go b/src/cmd/nncp-cfgnew/main.go index 8dad818..a1358ad 100644 --- a/src/cmd/nncp-cfgnew/main.go +++ b/src/cmd/nncp-cfgnew/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-check/main.go b/src/cmd/nncp-check/main.go index 252f79a..96ddd3f 100644 --- a/src/cmd/nncp-check/main.go +++ b/src/cmd/nncp-check/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-daemon/main.go b/src/cmd/nncp-daemon/main.go index 9d614f7..b7aceb7 100644 --- a/src/cmd/nncp-daemon/main.go +++ b/src/cmd/nncp-daemon/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -66,7 +66,7 @@ func (c InetdConn) Close() error { return c.w.Close() } -func performSP(ctx *nncp.Ctx, conn nncp.ConnDeadlined, nice uint8) { +func performSP(ctx *nncp.Ctx, conn nncp.ConnDeadlined, nice uint8) *nncp.SPState { state := nncp.SPState{ Ctx: ctx, Nice: nice, @@ -89,6 +89,7 @@ func performSP(ctx *nncp.Ctx, conn nncp.ConnDeadlined, nice uint8) { } ctx.LogE("call-start", nncp.SDS{"node": nodeId}, err, "") } + return &state } func main() { @@ -106,6 +107,13 @@ func main() { debug = flag.Bool("debug", false, "Print debug messages") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") + + autotoss = flag.Bool("autotoss", false, "Toss after call is finished") + autotossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autotossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") + autotossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") + autotossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") + autotossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") ) flag.Usage = usage flag.Parse() @@ -159,8 +167,20 @@ func main() { } ctx.LogD("daemon", nncp.SDS{"addr": conn.RemoteAddr()}, "accepted") go func(conn net.Conn) { - performSP(ctx, conn, nice) + state := performSP(ctx, conn, nice) conn.Close() // #nosec G104 + if *autotoss && state.Node != nil { + ctx.Toss( + state.Node.Id, + nice, + false, + *autotossDoSeen, + *autotossNoFile, + *autotossNoFreq, + *autotossNoExec, + *autotossNoTrns, + ) + } }(conn) } } diff --git a/src/cmd/nncp-exec/main.go b/src/cmd/nncp-exec/main.go index ddd3ffd..fe559ba 100644 --- a/src/cmd/nncp-exec/main.go +++ b/src/cmd/nncp-exec/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -37,6 +37,8 @@ func usage() { func main() { var ( + useTmp = flag.Bool("use-tmp", false, "Use temporary file, instead of memory buffer") + noCompress = flag.Bool("nocompress", false, "Do not compress input data") cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") niceRaw = flag.String("nice", nncp.NicenessFmt(nncp.DefaultNiceExec), "Outbound packet niceness") replyNiceRaw = flag.String("replynice", nncp.NicenessFmt(nncp.DefaultNiceFile), "Possible reply packet niceness") @@ -106,6 +108,8 @@ func main() { flag.Args()[2:], bufio.NewReader(os.Stdin), int64(*minSize)*1024, + *useTmp, + *noCompress, ); err != nil { log.Fatalln(err) } diff --git a/src/cmd/nncp-file/main.go b/src/cmd/nncp-file/main.go index 9a69e6c..1e1c176 100644 --- a/src/cmd/nncp-file/main.go +++ b/src/cmd/nncp-file/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-freq/main.go b/src/cmd/nncp-freq/main.go index 1bbcf98..65b06e4 100644 --- a/src/cmd/nncp-freq/main.go +++ b/src/cmd/nncp-freq/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-log/main.go b/src/cmd/nncp-log/main.go index 2535b89..7406261 100644 --- a/src/cmd/nncp-log/main.go +++ b/src/cmd/nncp-log/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-pkt/main.go b/src/cmd/nncp-pkt/main.go index 1264974..145b30b 100644 --- a/src/cmd/nncp-pkt/main.go +++ b/src/cmd/nncp-pkt/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -105,12 +105,14 @@ func main() { payloadType = "file request" case nncp.PktTypeExec: payloadType = "exec" + case nncp.PktTypeExecFat: + payloadType = "exec uncompressed" case nncp.PktTypeTrns: payloadType = "transitional" } var path string switch pkt.Type { - case nncp.PktTypeExec: + case nncp.PktTypeExec, nncp.PktTypeExecFat: path = string(bytes.Replace( pkt.Path[:pkt.PathLen], []byte{0}, diff --git a/src/cmd/nncp-reass/main.go b/src/cmd/nncp-reass/main.go index 8e1ba0b..8278700 100644 --- a/src/cmd/nncp-reass/main.go +++ b/src/cmd/nncp-reass/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/cmd/nncp-rm/main.go b/src/cmd/nncp-rm/main.go index 911db63..7afb24b 100644 --- a/src/cmd/nncp-rm/main.go +++ b/src/cmd/nncp-rm/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -24,7 +24,10 @@ import ( "log" "os" "path/filepath" + "regexp" + "strconv" "strings" + "time" "go.cypherpunks.ru/nncp/v5" ) @@ -38,6 +41,7 @@ func usage() { fmt.Fprintf(os.Stderr, " %s [options] -node NODE -seen\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s [options] -node NODE {-rx|-tx}\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s [options] -node NODE -pkt PKT\n", os.Args[0]) + fmt.Fprintln(os.Stderr, "-older option's time units are: (s)econds, (m)inutes, (h)ours, (d)ays") fmt.Fprintln(os.Stderr, "Options:") flag.PrintDefaults() } @@ -52,6 +56,8 @@ func main() { doTx = flag.Bool("tx", false, "Process transfered packets") doPart = flag.Bool("part", false, "Remove only .part files") doSeen = flag.Bool("seen", false, "Remove only .seen files") + older = flag.String("older", "", "XXX{smhd}: only older than XXX number of time units") + dryRun = flag.Bool("dryrun", false, "Do not actually remove files") pktRaw = flag.String("pkt", "", "Packet to remove") spoolPath = flag.String("spool", "", "Override path to spool") quiet = flag.Bool("quiet", false, "Print only errors") @@ -76,6 +82,31 @@ func main() { } ctx.Umask() + var oldBoundaryRaw int + if *older != "" { + olderRe := regexp.MustCompile(`^(\d+)([smhd])$`) + matches := olderRe.FindStringSubmatch(*older) + if len(matches) != 1+2 { + log.Fatalln("can not parse -older") + } + oldBoundaryRaw, err = strconv.Atoi(matches[1]) + if err != nil { + log.Fatalln("can not parse -older:", err) + } + switch matches[2] { + case "s": + break + case "m": + oldBoundaryRaw *= 60 + case "h": + oldBoundaryRaw *= 60 * 60 + case "d": + oldBoundaryRaw *= 60 * 60 * 24 + } + } + oldBoundary := time.Second * time.Duration(oldBoundaryRaw) + + now := time.Now() if *doTmp { err = filepath.Walk( filepath.Join(ctx.Spool, "tmp"), @@ -86,7 +117,14 @@ func main() { if info.IsDir() { return nil } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD("nncp-rm", nncp.SDS{"file": path}, "too fresh, skipping") + return nil + } ctx.LogI("nncp-rm", nncp.SDS{"file": path}, "") + if *dryRun { + return nil + } return os.Remove(path) }) if err != nil { @@ -104,6 +142,9 @@ func main() { } if strings.HasSuffix(info.Name(), ".lock") { ctx.LogI("nncp-rm", nncp.SDS{"file": path}, "") + if *dryRun { + return nil + } return os.Remove(path) } return nil @@ -131,16 +172,29 @@ func main() { if info.IsDir() { return nil } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD("nncp-rm", nncp.SDS{"file": path}, "too fresh, skipping") + return nil + } if *doSeen && strings.HasSuffix(info.Name(), nncp.SeenSuffix) { ctx.LogI("nncp-rm", nncp.SDS{"file": path}, "") + if *dryRun { + return nil + } return os.Remove(path) } if *doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix) { ctx.LogI("nncp-rm", nncp.SDS{"file": path}, "") + if *dryRun { + return nil + } return os.Remove(path) } if *pktRaw != "" && filepath.Base(info.Name()) == *pktRaw { ctx.LogI("nncp-rm", nncp.SDS{"file": path}, "") + if *dryRun { + return nil + } return os.Remove(path) } if !*doSeen && @@ -148,6 +202,9 @@ func main() { (*doRx || *doTx) && ((*doRx && xx == nncp.TRx) || (*doTx && xx == nncp.TTx)) { ctx.LogI("nncp-rm", nncp.SDS{"file": path}, "") + if *dryRun { + return nil + } return os.Remove(path) } return nil diff --git a/src/cmd/nncp-stat/main.go b/src/cmd/nncp-stat/main.go index fa33d4c..04aaed2 100644 --- a/src/cmd/nncp-stat/main.go +++ b/src/cmd/nncp-stat/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -111,7 +111,7 @@ func main() { for job := range ctx.Jobs(node.Id, nncp.TTx) { job.Fd.Close() // #nosec G104 if *showPkt { - jobPrint(nncp.TRx, job) + jobPrint(nncp.TTx, job) } txNums[job.PktEnc.Nice] = txNums[job.PktEnc.Nice] + 1 txBytes[job.PktEnc.Nice] = txBytes[job.PktEnc.Nice] + job.Size diff --git a/src/cmd/nncp-toss/main.go b/src/cmd/nncp-toss/main.go index 89b94d9..b26da3e 100644 --- a/src/cmd/nncp-toss/main.go +++ b/src/cmd/nncp-toss/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -43,10 +43,10 @@ func main() { dryRun = flag.Bool("dryrun", false, "Do not actually write any tossed data") doSeen = flag.Bool("seen", false, "Create .seen files") cycle = flag.Uint("cycle", 0, "Repeat tossing after N seconds in infinite loop") - noFile = flag.Bool("nofile", false, "Do not process packets with type: file") - noFreq = flag.Bool("nofreq", false, "Do not process packets with type: freq") - noExec = flag.Bool("noexec", false, "Do not process packets with type: exec") - noTrns = flag.Bool("notrns", false, "Do not process packets with type: transitional") + noFile = flag.Bool("nofile", false, "Do not process \"file\" packets") + noFreq = flag.Bool("nofreq", false, "Do not process \"freq\" packets") + noExec = flag.Bool("noexec", false, "Do not process \"exec\" packets") + noTrns = flag.Bool("notrns", false, "Do not process \"transitional\" packets") spoolPath = flag.String("spool", "", "Override path to spool") logPath = flag.String("log", "", "Override path to logfile") quiet = flag.Bool("quiet", false, "Print only errors") diff --git a/src/cmd/nncp-xfer/main.go b/src/cmd/nncp-xfer/main.go index 77cd9c6..50e9bf4 100644 --- a/src/cmd/nncp-xfer/main.go +++ b/src/cmd/nncp-xfer/main.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/ctx.go b/src/ctx.go index 9196930..645483d 100644 --- a/src/ctx.go +++ b/src/ctx.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/eblob.go b/src/eblob.go index e1fd012..c743d15 100644 --- a/src/eblob.go +++ b/src/eblob.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/go.mod b/src/go.mod index 7bfe83a..1b808cd 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,12 +6,12 @@ require ( github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/hjson/hjson-go v3.1.0+incompatible - github.com/klauspost/compress v1.11.0 + github.com/klauspost/compress v1.11.4 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect go.cypherpunks.ru/balloon v1.1.1 - golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a - golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa - golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d + golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad + golang.org/x/net v0.0.0-20201224014010-6772e930b67b + golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect ) diff --git a/src/go.sum b/src/go.sum index d1d7a3f..33b95ab 100644 --- a/src/go.sum +++ b/src/go.sum @@ -8,8 +8,8 @@ github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVf github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= github.com/hjson/hjson-go v3.1.0+incompatible h1:DY/9yE8ey8Zv22bY+mHV1uk2yRy0h8tKhZ77hEdi0Aw= github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= -github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg= -github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -18,15 +18,21 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA go.cypherpunks.ru/balloon v1.1.1 h1:ypHM1DRf/XuCrp9pDkTHg00CqZX/Np/APb//iHvDJTA= go.cypherpunks.ru/balloon v1.1.1/go.mod h1:k4s4ozrIrhpBjj78Z7LX8ZHxMQ+XE7DZUWl8gP2ojCo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d h1:L/IKR6COd7ubZrs2oTnTi73IhgqJ71c9s80WsQnh0Es= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 h1:/dSxr6gT0FNI1MO5WLJo8mTmItROeOKTkDn+7OwWBos= +golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/src/humanizer.go b/src/humanizer.go index 1eded3c..78e2fc1 100644 --- a/src/humanizer.go +++ b/src/humanizer.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/jobs.go b/src/jobs.go index d7363ed..2979064 100644 --- a/src/jobs.go +++ b/src/jobs.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/lockdir.go b/src/lockdir.go index 875a5bb..919d4cf 100644 --- a/src/lockdir.go +++ b/src/lockdir.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -51,6 +51,6 @@ func (ctx *Ctx) LockDir(nodeId *NodeId, lockCtx string) (*os.File, error) { func (ctx *Ctx) UnlockDir(fd *os.File) { if fd != nil { unix.Flock(int(fd.Fd()), unix.LOCK_UN) // #nosec G104 - fd.Close() // #nosec G104 + fd.Close() // #nosec G104 } } diff --git a/src/log.go b/src/log.go index 23a43d9..df2b491 100644 --- a/src/log.go +++ b/src/log.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/nncp.go b/src/nncp.go index 4431663..5fc6483 100644 --- a/src/nncp.go +++ b/src/nncp.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -38,7 +38,7 @@ along with this program. If not, see .` ) var ( - Version string = "5.4.1" + Version string = "5.5.0" Base32Codec *base32.Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) ) @@ -49,7 +49,7 @@ func VersionGet() string { func UsageHeader() string { return VersionGet() + ` -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev License GPLv3: GNU GPL version 3 This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. diff --git a/src/node.go b/src/node.go index 999af3d..c965796 100644 --- a/src/node.go +++ b/src/node.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/pipe.go b/src/pipe.go index a117bc8..ef25a91 100644 --- a/src/pipe.go +++ b/src/pipe.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/pkt.go b/src/pkt.go index 6eb4ef2..4dea1bf 100644 --- a/src/pkt.go +++ b/src/pkt.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -40,14 +40,17 @@ const ( EncBlkSize = 128 * (1 << 10) KDFXOFSize = chacha20poly1305.KeySize * 2 - PktTypeFile PktType = iota - PktTypeFreq PktType = iota - PktTypeExec PktType = iota - PktTypeTrns PktType = iota + PktTypeFile PktType = iota + PktTypeFreq PktType = iota + PktTypeExec PktType = iota + PktTypeTrns PktType = iota + PktTypeExecFat PktType = iota MaxPathSize = 1<<8 - 1 NNCPBundlePrefix = "NNCP" + + PktSizeOverhead = 8 + poly1305.TagSize ) var ( @@ -56,9 +59,8 @@ var ( BadMagic error = errors.New("Unknown magic number") BadPktType error = errors.New("Unknown packet type") - PktOverhead int64 - PktEncOverhead int64 - PktSizeOverhead int64 = 8 + poly1305.TagSize + PktOverhead int64 + PktEncOverhead int64 ) type Pkt struct { @@ -66,7 +68,7 @@ type Pkt struct { Type PktType Nice uint8 PathLen uint8 - Path *[MaxPathSize]byte + Path [MaxPathSize]byte } type PktTbs struct { @@ -74,7 +76,7 @@ type PktTbs struct { Nice uint8 Sender *NodeId Recipient *NodeId - ExchPub *[32]byte + ExchPub [32]byte } type PktEnc struct { @@ -82,15 +84,12 @@ type PktEnc struct { Nice uint8 Sender *NodeId Recipient *NodeId - ExchPub *[32]byte - Sign *[ed25519.SignatureSize]byte + ExchPub [32]byte + Sign [ed25519.SignatureSize]byte } func init() { - pkt := Pkt{ - Type: PktTypeFile, - Path: new([MaxPathSize]byte), - } + pkt := Pkt{Type: PktTypeFile} var buf bytes.Buffer n, err := xdr.Marshal(&buf, pkt) if err != nil { @@ -105,11 +104,8 @@ func init() { } pktEnc := PktEnc{ Magic: MagicNNCPEv4, - Nice: 123, Sender: dummyId, Recipient: dummyId, - ExchPub: new([32]byte), - Sign: new([ed25519.SignatureSize]byte), } n, err = xdr.Marshal(&buf, pktEnc) if err != nil { @@ -127,7 +123,6 @@ func NewPkt(typ PktType, nice uint8, path []byte) (*Pkt, error) { Type: typ, Nice: nice, PathLen: uint8(len(path)), - Path: new([MaxPathSize]byte), } copy(pkt.Path[:], path) return &pkt, nil @@ -211,7 +206,7 @@ func PktEncWrite( Nice: nice, Sender: our.Id, Recipient: their.Id, - ExchPub: pubEph, + ExchPub: *pubEph, } var tbsBuf bytes.Buffer if _, err = xdr.Marshal(&tbsBuf, &tbs); err != nil { @@ -224,8 +219,8 @@ func PktEncWrite( Nice: nice, Sender: our.Id, Recipient: their.Id, - ExchPub: pubEph, - Sign: signature, + ExchPub: *pubEph, + Sign: *signature, } if _, err = xdr.Marshal(out, &pktEnc); err != nil { return err @@ -325,7 +320,7 @@ func PktEncRead( return their, 0, errors.New("Invalid signature") } sharedKey := new([32]byte) - curve25519.ScalarMult(sharedKey, our.ExchPrv, pktEnc.ExchPub) + curve25519.ScalarMult(sharedKey, our.ExchPrv, &pktEnc.ExchPub) kdf, err := blake2b.NewXOF(KDFXOFSize, sharedKey[:]) if err != nil { return their, 0, err diff --git a/src/pkt_test.go b/src/pkt_test.go index d076b63..cbf7818 100644 --- a/src/pkt_test.go +++ b/src/pkt_test.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/progress.go b/src/progress.go index 546979a..61ad465 100644 --- a/src/progress.go +++ b/src/progress.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/sp.go b/src/sp.go index cff8385..72ae7fb 100644 --- a/src/sp.go +++ b/src/sp.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -56,7 +56,7 @@ var ( DefaultDeadline = 10 * time.Second PingTimeout = time.Minute - spWorkersGroup sync.WaitGroup + spCheckerToken chan struct{} ) type SPType uint8 @@ -148,6 +148,8 @@ func init() { panic(err) } SPFileOverhead = buf.Len() + spCheckerToken = make(chan struct{}, 1) + spCheckerToken <- struct{}{} } func MarshalSP(typ SPType, sp interface{}) []byte { @@ -989,9 +991,11 @@ func (state *SPState) ProcessSP(payload []byte) ([][]byte, error) { fd.Close() // #nosec G104 continue } - spWorkersGroup.Wait() - spWorkersGroup.Add(1) + <-spCheckerToken go func() { + defer func() { + spCheckerToken <- struct{}{} + }() if err := fd.Sync(); err != nil { state.Ctx.LogE("sp-file", sdsp, err, "sync") fd.Close() // #nosec G104 @@ -1023,7 +1027,6 @@ func (state *SPState) ProcessSP(payload []byte) ([][]byte, error) { state.Lock() delete(state.infosTheir, *file.Hash) state.Unlock() - spWorkersGroup.Done() state.wg.Add(1) go func() { state.payloads <- MarshalSP(SPTypeDone, SPDone{file.Hash}) diff --git a/src/tmp.go b/src/tmp.go index 62ec5bb..f1be071 100644 --- a/src/tmp.go +++ b/src/tmp.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -74,8 +74,8 @@ func (ctx *Ctx) NewTmpFileWHash() (*TmpFileWHash, error) { } func (tmp *TmpFileWHash) Cancel() { - tmp.Fd.Truncate(0) // #nosec G104 - tmp.Fd.Close() // #nosec G104 + tmp.Fd.Truncate(0) // #nosec G104 + tmp.Fd.Close() // #nosec G104 os.Remove(tmp.Fd.Name()) // #nosec G104 } diff --git a/src/toss.go b/src/toss.go index c0aa7bf..f70ffd3 100644 --- a/src/toss.go +++ b/src/toss.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -126,7 +126,7 @@ func (ctx *Ctx) Toss( sds["size"] = pktSize ctx.LogD("rx", sds, "taken") switch pkt.Type { - case PktTypeExec: + case PktTypeExec, PktTypeExecFat: if noExec { goto Closing } @@ -148,8 +148,10 @@ func (ctx *Ctx) Toss( isBad = true goto Closing } - if err = decompressor.Reset(pipeR); err != nil { - log.Fatalln(err) + if pkt.Type == PktTypeExec { + if err = decompressor.Reset(pipeR); err != nil { + log.Fatalln(err) + } } if !dryRun { cmd := exec.Command( @@ -162,7 +164,11 @@ func (ctx *Ctx) Toss( "NNCP_SENDER="+sender.Id.String(), "NNCP_NICE="+strconv.Itoa(int(pkt.Nice)), ) - cmd.Stdin = decompressor + if pkt.Type == PktTypeExec { + cmd.Stdin = decompressor + } else { + cmd.Stdin = pipeR + } output, err := cmd.Output() if err != nil { ctx.LogE("rx", sds, err, "handle") diff --git a/src/toss_test.go b/src/toss_test.go index a76615d..8a16dd9 100644 --- a/src/toss_test.go +++ b/src/toss_test.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -99,6 +99,8 @@ func TestTossExec(t *testing.T) { []string{"arg0", "arg1"}, strings.NewReader("BODY\n"), 1<<15, + false, + false, ); err != nil { t.Error(err) return false @@ -449,7 +451,6 @@ func TestTossTrns(t *testing.T) { Magic: MagicNNCPPv3, Type: PktTypeTrns, PathLen: blake2b.Size256, - Path: new([MaxPathSize]byte), } copy(pktTrans.Path[:], nodeOur.Id[:]) var dst bytes.Buffer diff --git a/src/tx.go b/src/tx.go index 4c31d92..3352f86 100644 --- a/src/tx.go +++ b/src/tx.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -131,49 +131,63 @@ type DummyCloser struct{} func (dc DummyCloser) Close() error { return nil } -func prepareTxFile(srcPath string) (reader io.Reader, closer io.Closer, fileSize int64, archived bool, rerr error) { - if srcPath == "-" { - // Read content from stdin, saving to temporary file, encrypting - // on the fly - src, err := ioutil.TempFile("", "nncp-file") - if err != nil { - rerr = err - return - } - os.Remove(src.Name()) // #nosec G104 - tmpW := bufio.NewWriter(src) - tmpKey := make([]byte, chacha20poly1305.KeySize) - if _, rerr = rand.Read(tmpKey[:]); rerr != nil { - return - } - aead, err := chacha20poly1305.New(tmpKey) - if err != nil { - rerr = err - return - } - nonce := make([]byte, aead.NonceSize()) - written, err := aeadProcess(aead, nonce, true, bufio.NewReader(os.Stdin), tmpW) - if err != nil { - rerr = err - return - } - fileSize = int64(written) - if err = tmpW.Flush(); err != nil { - rerr = err - return - } - if _, err = src.Seek(0, io.SeekStart); err != nil { - rerr = err - return +func throughTmpFile(r io.Reader) ( + reader io.Reader, + closer io.Closer, + fileSize int64, + rerr error, +) { + src, err := ioutil.TempFile("", "nncp-file") + if err != nil { + rerr = err + return + } + os.Remove(src.Name()) // #nosec G104 + tmpW := bufio.NewWriter(src) + tmpKey := make([]byte, chacha20poly1305.KeySize) + if _, rerr = rand.Read(tmpKey[:]); rerr != nil { + return + } + aead, err := chacha20poly1305.New(tmpKey) + if err != nil { + rerr = err + return + } + nonce := make([]byte, aead.NonceSize()) + written, err := aeadProcess(aead, nonce, true, r, tmpW) + if err != nil { + rerr = err + return + } + fileSize = int64(written) + if err = tmpW.Flush(); err != nil { + rerr = err + return + } + if _, err = src.Seek(0, io.SeekStart); err != nil { + rerr = err + return + } + r, w := io.Pipe() + go func() { + if _, err := aeadProcess(aead, nonce, false, bufio.NewReader(src), w); err != nil { + w.CloseWithError(err) // #nosec G104 } - r, w := io.Pipe() - go func() { - if _, err := aeadProcess(aead, nonce, false, bufio.NewReader(src), w); err != nil { - w.CloseWithError(err) // #nosec G104 - } - }() - reader = r - closer = src + }() + reader = r + closer = src + return +} + +func prepareTxFile(srcPath string) ( + reader io.Reader, + closer io.Closer, + fileSize int64, + archived bool, + rerr error, +) { + if srcPath == "-" { + reader, closer, fileSize, rerr = throughTmpFile(bufio.NewReader(os.Stdin)) return } @@ -481,33 +495,94 @@ func (ctx *Ctx) TxExec( args []string, in io.Reader, minSize int64, + useTmp bool, + noCompress bool, ) error { path := make([][]byte, 0, 1+len(args)) path = append(path, []byte(handle)) for _, arg := range args { path = append(path, []byte(arg)) } - pkt, err := NewPkt(PktTypeExec, replyNice, bytes.Join(path, []byte{0})) - if err != nil { - return err + pktType := PktTypeExec + if noCompress { + pktType = PktTypeExecFat } - var compressed bytes.Buffer - compressor, err := zstd.NewWriter( - &compressed, - zstd.WithEncoderLevel(zstd.SpeedDefault), - ) + pkt, err := NewPkt(pktType, replyNice, bytes.Join(path, []byte{0})) if err != nil { return err } - if _, err = io.Copy(compressor, in); err != nil { - compressor.Close() // #nosec G104 - return err + var size int64 + + if !noCompress && !useTmp { + var compressed bytes.Buffer + compressor, err := zstd.NewWriter( + &compressed, + zstd.WithEncoderLevel(zstd.SpeedDefault), + ) + if err != nil { + return err + } + if _, err = io.Copy(compressor, in); err != nil { + compressor.Close() // #nosec G104 + return err + } + if err = compressor.Close(); err != nil { + return err + } + size = int64(compressed.Len()) + _, err = ctx.Tx(node, pkt, nice, size, minSize, &compressed, handle) } - if err = compressor.Close(); err != nil { - return err + if noCompress && !useTmp { + var data bytes.Buffer + if _, err = io.Copy(&data, in); err != nil { + return err + } + size = int64(data.Len()) + _, err = ctx.Tx(node, pkt, nice, size, minSize, &data, handle) } - size := int64(compressed.Len()) - _, err = ctx.Tx(node, pkt, nice, size, minSize, &compressed, handle) + if !noCompress && useTmp { + r, w := io.Pipe() + compressor, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return err + } + copyErr := make(chan error) + go func() { + _, err := io.Copy(compressor, in) + if err != nil { + compressor.Close() // #nosec G104 + copyErr <- err + } + err = compressor.Close() + w.Close() + copyErr <- err + }() + tmpReader, closer, fileSize, err := throughTmpFile(r) + if closer != nil { + defer closer.Close() + } + if err != nil { + return err + } + err = <-copyErr + if err != nil { + return err + } + size = fileSize + _, err = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle) + } + if noCompress && useTmp { + tmpReader, closer, fileSize, err := throughTmpFile(in) + if closer != nil { + defer closer.Close() + } + if err != nil { + return err + } + size = fileSize + _, err = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle) + } + sds := SDS{ "type": "exec", "node": node.Id, diff --git a/src/tx_test.go b/src/tx_test.go index 9a16d9a..ab0c1c6 100644 --- a/src/tx_test.go +++ b/src/tx_test.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/via.go b/src/via.go index 251af21..32d7aac 100644 --- a/src/via.go +++ b/src/via.go @@ -1,6 +1,6 @@ /* NNCP -- Node to Node copy, utilities for store-and-forward data exchange -Copyright (C) 2016-2020 Sergey Matveev +Copyright (C) 2016-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/test.do b/test.do index 7f3c01e..feb7b5f 100644 --- a/test.do +++ b/test.do @@ -1,4 +1,5 @@ redo-ifchange config gopath module-name +exec >&2 . ./config . ./gopath mod=`cat module-name`