From ab7c7eca0e53661f0ba904c2a6ba752990bea367 Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Thu, 1 Jul 2021 10:54:55 +0300 Subject: [PATCH] Multicast areas --- README | 2 +- README.RU | 2 +- doc/about.ru.texi | 1 + doc/about.texi | 1 + doc/call.texi | 2 +- doc/cfg.texi | 239 ----- doc/cfg/areas.texi | 67 ++ doc/cfg/general.texi | 48 + doc/cfg/index.texi | 49 + doc/cfg/neigh.texi | 160 ++++ doc/cfg/notify.texi | 42 + doc/cfg/self.texi | 10 + doc/cmd/index.texi | 104 +++ doc/cmd/nncp-bundle.texi | 46 + doc/cmd/nncp-call.texi | 60 ++ doc/cmd/nncp-caller.texi | 15 + doc/cmd/nncp-cfgenc.texi | 39 + doc/cmd/nncp-cfgmin.texi | 11 + doc/cmd/nncp-cfgnew.texi | 17 + doc/cmd/nncp-check.texi | 18 + doc/cmd/nncp-cronexpr.texi | 24 + doc/cmd/nncp-daemon.texi | 36 + doc/cmd/nncp-exec.texi | 51 + doc/cmd/nncp-file.texi | 56 ++ doc/cmd/nncp-freq.texi | 15 + doc/cmd/nncp-hash.texi | 17 + doc/cmd/nncp-log.texi | 9 + doc/cmd/nncp-pkt.texi | 44 + doc/cmd/nncp-reass.texi | 58 ++ doc/cmd/nncp-rm.texi | 53 ++ doc/cmd/nncp-stat.texi | 13 + doc/cmd/nncp-toss.texi | 32 + doc/cmd/nncp-xfer.texi | 31 + doc/cmds.texi | 656 ------------- doc/comparison.ru.texi | 1 + doc/comparison.texi | 1 + doc/index.texi | 8 +- doc/multicast.texi | 148 +++ doc/news.ru.texi | 22 + doc/news.texi | 21 + doc/pkt/area.texi | 23 + doc/{pkt.texi => pkt/encrypted.texi} | 73 -- doc/pkt/index.texi | 15 + doc/pkt/plain.texi | 144 +++ makedist.sh | 4 +- ports/nncp/Makefile | 2 +- ports/nncp/pkg-descr | 2 +- src/area.go | 69 ++ src/call.go | 1 + src/cfg.go | 75 ++ src/cmd/nncp-call/main.go | 2 + src/cmd/nncp-caller/main.go | 2 + src/cmd/nncp-cfgnew/main.go | 75 +- src/cmd/nncp-daemon/main.go | 3 + src/cmd/nncp-exec/main.go | 22 +- src/cmd/nncp-file/main.go | 33 +- src/cmd/nncp-pkt/main.go | 97 +- src/cmd/nncp-rm/main.go | 173 ++-- src/cmd/nncp-toss/main.go | 19 +- src/ctx.go | 3 + src/jobs.go | 5 +- src/magic.go | 4 + src/nncp.go | 2 +- src/node.go | 2 + src/pkt.go | 71 +- src/pkt_test.go | 2 +- src/toss.go | 1284 ++++++++++++++++---------- src/toss_test.go | 35 +- src/tx.go | 177 +++- src/tx_test.go | 5 +- 70 files changed, 2979 insertions(+), 1674 deletions(-) delete mode 100644 doc/cfg.texi create mode 100644 doc/cfg/areas.texi create mode 100644 doc/cfg/general.texi create mode 100644 doc/cfg/index.texi create mode 100644 doc/cfg/neigh.texi create mode 100644 doc/cfg/notify.texi create mode 100644 doc/cfg/self.texi create mode 100644 doc/cmd/index.texi create mode 100644 doc/cmd/nncp-bundle.texi create mode 100644 doc/cmd/nncp-call.texi create mode 100644 doc/cmd/nncp-caller.texi create mode 100644 doc/cmd/nncp-cfgenc.texi create mode 100644 doc/cmd/nncp-cfgmin.texi create mode 100644 doc/cmd/nncp-cfgnew.texi create mode 100644 doc/cmd/nncp-check.texi create mode 100644 doc/cmd/nncp-cronexpr.texi create mode 100644 doc/cmd/nncp-daemon.texi create mode 100644 doc/cmd/nncp-exec.texi create mode 100644 doc/cmd/nncp-file.texi create mode 100644 doc/cmd/nncp-freq.texi create mode 100644 doc/cmd/nncp-hash.texi create mode 100644 doc/cmd/nncp-log.texi create mode 100644 doc/cmd/nncp-pkt.texi create mode 100644 doc/cmd/nncp-reass.texi create mode 100644 doc/cmd/nncp-rm.texi create mode 100644 doc/cmd/nncp-stat.texi create mode 100644 doc/cmd/nncp-toss.texi create mode 100644 doc/cmd/nncp-xfer.texi delete mode 100644 doc/cmds.texi create mode 100644 doc/multicast.texi create mode 100644 doc/pkt/area.texi rename doc/{pkt.texi => pkt/encrypted.texi} (61%) create mode 100644 doc/pkt/index.texi create mode 100644 doc/pkt/plain.texi create mode 100644 src/area.go diff --git a/README b/README index eca5b5a..9aa5a78 100644 --- a/README +++ b/README @@ -8,7 +8,7 @@ requests, Internet mail and commands transmission. All packets are integrity checked, end-to-end encrypted (E2EE), explicitly authenticated by known participants public keys. Onion encryption is applied to relayed packets. Each node acts both as a client and server, can use -push and poll behaviour model. +push and poll behaviour model. Also there is multicasting areas support. Out-of-box offline sneakernet/floppynet, dead drops, sequential and append-only CD-ROM/tape storages, air-gapped computers support. But diff --git a/README.RU b/README.RU index 9da6599..a48bc31 100644 --- a/README.RU +++ b/README.RU @@ -10,7 +10,7 @@ NNCP (Node to Node copy) это набор утилит упрощающий б ключами участников. Луковичное (onion) шифрование применяется ко всем ретранслируемым пакетам. Каждый узел выступает одновременно в роли клиента и сервера, может использовать как push, так и poll модель -поведения. +поведения. А также есть поддержка мультивещательной рассылки пакетов. Поддержка из коробки offline флоппинета, тайников для сброса информации (dead drop), последовательных и не перезаписываемых CD-ROM/ленточных diff --git a/doc/about.ru.texi b/doc/about.ru.texi index bb9c2d0..bd47b1a 100644 --- a/doc/about.ru.texi +++ b/doc/about.ru.texi @@ -20,6 +20,7 @@ (onion) шифрование применяется ко всем ретранслируемым пакетам. Каждый узел выступает одновременно в роли клиента и сервера, может использовать как push, так и poll модель поведения. +А также есть поддержка @ref{Multicast, мультивещательной} рассылки пакетов. Поддержка из коробки offline @url{https://ru.wikipedia.org/wiki/%D0%A4%D0%BB%D0%BE%D0%BF%D0%BF%D0%B8%D0%BD%D0%B5%D1%82, diff --git a/doc/about.texi b/doc/about.texi index f228d15..f752931 100644 --- a/doc/about.texi +++ b/doc/about.texi @@ -15,6 +15,7 @@ end-to-end} encrypted, explicitly authenticated by known participants public keys. @url{https://en.wikipedia.org/wiki/Onion_routing, Onion encryption} is applied to relayed packets. Each node acts both as a client and server, can use push and poll behaviour model. +Also there is @ref{Multicast, multicasting} areas support. Out-of-box offline @url{https://en.wikipedia.org/wiki/Sneakernet, sneakernet/floppynet}, @url{https://en.wikipedia.org/wiki/Dead_drop, diff --git a/doc/call.texi b/doc/call.texi index 86f4bcf..83f43ea 100644 --- a/doc/call.texi +++ b/doc/call.texi @@ -1,7 +1,7 @@ @node Call @unnumbered Call configuration -Call is a rule when and how node can be called. +Call is a rule when and how node can be called by @ref{nncp-caller}. Example list of call structures: diff --git a/doc/cfg.texi b/doc/cfg.texi deleted file mode 100644 index 30341f8..0000000 --- a/doc/cfg.texi +++ /dev/null @@ -1,239 +0,0 @@ -@node Configuration -@unnumbered Configuration file - -Example @url{https://hjson.org/, Hjson} configuration file: - -@verbatim -{ - spool: /var/spool/nncp - log: /var/spool/nncp/log - umask: "022" - noprogress: true - nohdr: true - - mcd-listen: ["em0", "igb1"] - mcd-send: {em0: 60, igb1: 5} - - notify: { - file: { - from: nncp@localhost - to: user+file@example.com - } - freq: { - from: nncp@localhost - to: user+freq@example.com - } - exec: { - "*.warcer": { - from: nncp@localhost - to: user+warcer@example.com - } - "eve.warcer": { - from: nncp@localhost - to: user+warcer-overriden@example.com - } - } - } - - self: { - id: TIJQL...2NGIA - exchpub: CYVGQ...PSEWQ - exchprv: 65PUY...MPZ3Q - signpub: 2NMVC...CMH5Q - signprv: 555JD...RGD6Y - noiseprv: D62XU...NKYPA - noisepub: KIBKK...ESM7Q - } - - neigh: { - self: { - id: TIJQL...2NGIA - exchpub: CYVGQ...PSEWQ - signpub: 2NMVC...CMH5Q - noisepub: KIBKK...ESM7Q - exec: {sendmail: ["/usr/sbin/sendmail"]} - } - alice: { - id: "XJZBK...65IJQ" - exchpub: MJACJ...FAI6A - signpub: T4AFC...N2FRQ - noisepub: UBM5K...VI42A - exec: {flag: ["/usr/bin/touch", "-t"]} - incoming: "/home/alice/incoming" - onlinedeadline: 1800 - maxonlinetime: 3600 - addrs: { - lan: "[fe80::1234%igb0]:5400" - internet: alice.com:3389 - proxied: "|ssh remote.host nncp-daemon -inetd" - } - calls: [ - { - cron: "*/2 * * * *" - }, - ] - } - bob: { - id: 2IZNP...UYGYA - exchpub: WFLMZ...B7NHA - signpub: GTGXG...IE3OA - exec: { - sendmail: ["/usr/sbin/sendmail"] - warcer: ["/path/to/warcer.sh"] - wgeter: ["/path/to/wgeter.sh"] - } - freq: { - path: "/home/bob/pub" - chunked: 1024 - minsize: 2048 - } - via: ["alice"] - rxrate: 10 - txrate: 20 - } - } -} -@end verbatim - -@strong{spool} field contains an absolute path to @ref{Spool, spool} -directory. @strong{log} field contains an absolute path to @ref{Log, -log} file. - -Non-empty optional @strong{umask} will force all invoked commands to -override their umask to specified octal mask. Useful for using with -@ref{Shared spool, shared spool directories}. - -Enabled @strong{noprogress} option disabled progress showing for many -commands by default. You can always force its showing with -@option{-progress} command line option anyway. - -@anchor{CfgNoHdr} -@strong{nohdr} option disables @ref{HdrFile, .hdr} files usage. - -@anchor{CfgMCDListen} -@strong{mcd-listen} specifies list of network interfaces -@ref{nncp-caller} will listen for incoming @ref{MCD} announcements. - -@anchor{CfgMCDSend} -@strong{mcd-send} specifies list of network interfaces, and intervals in -seconds, where @ref{nncp-daemon} will send @ref{MCD} announcements. - -@anchor{CfgNotify} -@strong{notify} section contains notification settings for successfully -tossed file, freq and exec packets. Corresponding @strong{from} and -@strong{to} fields will be substituted in notification email message. -@code{neigh.self.exec.sendmail} will be used as a local mailer. You can -omit either of those two @code{from}/@code{to} sections to omit -corresponding notifications, or the whole section at once. - -@code{notify.exec} section is a mapping of exec handles and -corresponding @code{from}/@code{to} sections. Each handle has either -@code{NODE.HANDLE} or @code{*.HANDLE} syntax. You can override -notification options for some node with the first type of name. -Handle command's output will be included in notification messages. - -@strong{self} section contains our node's private keypairs. -@strong{exch*} and @strong{sign*} are used during @ref{Encrypted, -encrypted} packet creation. @strong{noise*} are used during @ref{Sync, -synchronization protocol} working in @ref{nncp-call}/@ref{nncp-daemon}. - -@strong{neigh} section contains all known neighbours information. It -always has @strong{self} neighbour that is copy of our node's public -data (public keys). It is useful for copy-paste sharing with your -friends. Each section's key is a human-readable name of the neighbour. - -Except for @code{id}, @code{exchpub} and @code{signpub} each neighbour -node has the following fields: - -@table @strong - -@item noisepub -If present, then node can be online called using @ref{Sync, -synchronization protocol}. Contains authentication public key. - -@anchor{CfgExec} -@item exec -Dictionary consisting of handles and corresponding command line -arguments. In example above there are @command{sendmail} handles, -@command{warcer}, @command{wgeter} and @command{flag} one. Remote node -can queue some handle execution with providing additional command line -arguments and the body fed to command's @code{stdin}. - -@verb{|sendmail: ["/usr/sbin/sendmail", "-t"]|} handle, when called by -@verb{|echo hello world | nncp-exec OURNODE sendmail ARG0 ARG1 ARG2|} -command, will execute: - -@example -NNCP_SELF=OURNODE \ -NNCP_SENDER=REMOTE \ -NNCP_NICE=64 \ -/usr/sbin/sendmail -t ARG0 ARG1 ARG2 -@end example - -feeding @verb{|hello world\n|} to that started @command{sendmail} -process. - -@anchor{CfgIncoming} -@item incoming -Full path to directory where all file uploads will be saved. May be -omitted to forbid file uploading on that node. - -@anchor{CfgFreq} -@item freq.path -Full path to directory from where file requests will queue files for -transmission. May be omitted to forbid freqing from that node. - -@item freq.chunked -If set, then enable @ref{Chunked, chunked} file transmission during -freqing. This is the desired chunk size in KiBs. - -@item freq.minsize -If set, then apply @ref{OptMinSize, -minsize} option during file -transmission. - -@anchor{CfgVia} -@item via -An array of node identifiers that will be used as a relay to that node. -For example @verb{|["foo","bar"]|} means that packet can reach current -node by transitioning through @code{foo} and then @code{bar} nodes. May -be omitted if direct connection exists and no relaying is required. - -@anchor{CfgAddrs} -@item addrs -Dictionary containing known network addresses of the node. Each key is -human-readable name of the address. For direct TCP connections use -@verb{|host:port|} format, pointing to @ref{nncp-daemon}'s listening -instance. Also you can pipe connection through the external command -using @verb{#|some command#} format. @code{/bin/sh -c "some command"} -will start and its @code{stdin}/@code{stdout} used as a connection. May -be omitted if either no direct connection exists, or @ref{nncp-call} is -used with forced address specifying. - -@anchor{CfgXxRate} -@item rxrate/txrate -If greater than zero, then at most *rate packets per second will be -sent/received after the handshake. It could be used as crude bandwidth -traffic shaper: each packet has at most 64 KiB payload size. Could be -omitted at all -- no rate limits. - -@anchor{CfgOnlineDeadline} -@item onlinedeadline -Online connection deadline of nodes inactivity in seconds. It is the -time connection considered dead after not receiving/sending any packets -(except for PINGs) and connection must be terminated. By default it is -set to 10 seconds. This can be set to rather high values to keep -connection alive (to reduce handshake overhead and delays), wait for -appearing packets ready to send and notifying remote side about their -appearance. - -@anchor{CfgMaxOnlineTime} -@item maxonlinetime -If greater than zero, then it is maximal time of single connection. -Forcefully disconnect if it is exceeded. - -@anchor{CfgCalls} -@item calls -List of @ref{Call, call configuration}s. Can be omitted if -@ref{nncp-caller} won't be used to call that node. - -@end table diff --git a/doc/cfg/areas.texi b/doc/cfg/areas.texi new file mode 100644 index 0000000..bc21008 --- /dev/null +++ b/doc/cfg/areas.texi @@ -0,0 +1,67 @@ +@node CfgAreas +@section Configuration areas options + +@ref{Multicast} areas configuration only used with multicast packets. + +@verbatim +areas: { + nodelist: { + id: OU67K7NA3RPOPFKJWNVBYJ5GPLRBDGHH6DZSSJ32JL7Q3Q76E52A + + pub: ALCX2NJBANMBNFTQ27C3C6W2WJIXSE74R27TSYZQKMD2UJERCEOQ + prv: VQ3B4TLAZZB2G7RS3OSS5NUVKAS44OGY5YMQPMTAHQMZZLNG25MA + + subs: ["alice", "bob", "eve"] + incoming: /home/incoming/areas/nodelist + } + echoarea: { + id: CKKJ3HOAVOP7VPNCEGZRNDO34MUOOJ4AXHDFCSVSOE647KN5CMIA + + pub: 5MFPTJI2R322EUCTGCWZXTDBCVEL5NCFDBXI5PHPQOTLUVSQ3ZIQ + prv: LVGIZQRQTDE524KEE5FOWLE2GCQBILY4VSQBDHWJC6YUTOJ54QCQ + + subs: ["alice", "bob"] + exec: {sendmail: ["/usr/sbin/sendmail"]} + allow-unknown: true + } + whatever.pvt: { + id: OU67K7NA3RPOPFKJWNVBYJ5GPLRBDGHH6DZSSJ32JL7Q3Q76E52A + subs: ["dave", "eve"] + } +} +@end verbatim + +Each key is human readable multicast group/area/echo name. + +The only required field is the @code{id}. You can not process multicast +packets that has unknown area identification. + +@code{subs} contains a list of recipients you must relay incoming +multicast packet on. + +Knowledge of @code{pub} and @code{prv} keys gives ability to decrypt +multicast packet and process its contents (file or exec transmission). +For accepting file transmissions you must set @code{incoming}, similar +to @ref{CfgIncoming, neigh's node option}. For accepting exec +transmissions you must set @code{exec}, similar to @ref{CfgExec, neigh's +node option}. + +You can accept multicast packets from unknown senders, by setting +@code{allow-unknown} option. + +In the example above: + +@table @code +@item nodelist +That area is for multicast sending of @file{nodelist} files, with +relaying it to @code{alice}, @code{bob} and @code{eve} further. +@item echoarea +That area is for multicast discussion through @code{sendmail} handled +exec packets. Relaying to @code{alice} and @code{bob} and accepting +messages from unknown participants. +@item whatever.pvt +We just relay that area packets to @code{dave} and @code{eve}, but +without ability to see what is inside them. Pay attention that +@code{allow-unknown} does not play any role here, because we are not +even trying to decrypt (and authenticate) those multicast packets. +@end table diff --git a/doc/cfg/general.texi b/doc/cfg/general.texi new file mode 100644 index 0000000..63d0757 --- /dev/null +++ b/doc/cfg/general.texi @@ -0,0 +1,48 @@ +@node CfgGeneral +@section Configuration general options + +Those options are in the root of configuration dictionary. + +@verbatim +spool: /var/spool/nncp +log: /var/spool/nncp/log + +# All of options below are optional +umask: "022" +noprogress: true +nohdr: true + +# MultiCast Discovery +mcd-listen: ["em0", "igb1"] +mcd-send: {em0: 60, igb1: 5} +@end verbatim + +@table @code +@item spool +Absolute path to the @ref{Spool, spool} directory. +@item log +Absolute path to the @ref{Log, log} file. +@item umask +Will force all invoked commands to override their umask to specified +octal mask. Useful for using with @ref{Shared spool, shared spool directories}. +@item noprogress +When enabled, disables progress showing for many commands by default. +You can always force its showing with @option{-progress} command line +option anyway. +@anchor{CfgNoHdr} +@item nohdr +@strong{nohdr} option disables @ref{HdrFile, .hdr} files usage. +@end table + +And optional @ref{MCD, MultiCast Discovery} options: + +@table @code +@anchor{CfgMCDListen} +@item mcd-listen +Specifies list of network interfaces @ref{nncp-caller} will listen for +incoming @ref{MCD} announcements. +@anchor{CfgMCDSend} +@item mcd-send +Specifies list of network interfaces, and intervals in seconds, where +@ref{nncp-daemon} will send @ref{MCD} announcements. +@end table diff --git a/doc/cfg/index.texi b/doc/cfg/index.texi new file mode 100644 index 0000000..1944eaa --- /dev/null +++ b/doc/cfg/index.texi @@ -0,0 +1,49 @@ +@node Configuration +@unnumbered Configuration file + +NNCP uses single file configuration file in @url{https://hjson.org/, +Hjson} format. Initially it is created with @ref{nncp-cfgnew} command +and at minimum it can look like this: + +@verbatim +spool: /var/spool/nncp +log: /var/spool/nncp/log + +self: { + id: RKOLY...KAMXQ + exchpub: 2NZKH...CMI7A + exchprv: KETPP...2OJZA + signpub: EXD7M...YAOFA + signprv: B3EMS..XMAHCQ + noiseprv: 3TJDF...2D7DQ + noisepub: MIXYN...BGNDQ +} + +neigh: { + self: { + id: RKOLY...KAMXQ + exchpub: 2NZKH...CMI7A + signpub: EXD7M...YAOFA + noisepub: MIXYN...BGNDQ + } +} +@end verbatim + +And for being able to communicate with at least one other node, you just +need to add single key to the @code{neigh} section similar to the "self". + +All configuration file can be separated on five sections: + +@menu +* General options: CfgGeneral. +* Self-node keypairs: CfgSelf. +* Notifications: CfgNotify. +* Neighbours: CfgNeigh. +* Areas: CfgAreas. +@end menu + +@include cfg/general.texi +@include cfg/self.texi +@include cfg/notify.texi +@include cfg/neigh.texi +@include cfg/areas.texi diff --git a/doc/cfg/neigh.texi b/doc/cfg/neigh.texi new file mode 100644 index 0000000..53eed46 --- /dev/null +++ b/doc/cfg/neigh.texi @@ -0,0 +1,160 @@ +@node CfgNeigh +@section Configuration neighbour options + +@strong{neigh} section contains all known neighbours information. It +always has @strong{self} neighbour that is copy of our node's public +data (public keys). It is useful for copy-paste sharing with your +friends. Each section's key is a human-readable name of the neighbour. + +@verbatim +neigh: { + self: { + id: RKOLY...KAMXQ + exchpub: 2NZKH...CMI7A + signpub: EXD7M...YAOFA + noisepub: MIXYN...BGNDQ + exec: {sendmail: ["/usr/sbin/sendmail"]} + } + alice: { + id: "XJZBK...65IJQ" + exchpub: MJACJ...FAI6A + signpub: T4AFC...N2FRQ + noisepub: UBM5K...VI42A + exec: {flag: ["/usr/bin/touch", "-t"]} + incoming: "/home/alice/incoming" + onlinedeadline: 1800 + maxonlinetime: 3600 + addrs: { + lan: "[fe80::1234%igb0]:5400" + internet: alice.com:3389 + proxied: "|ssh remote.host nncp-daemon -inetd" + } + calls: [ + { + cron: "*/2 * * * *" + }, + ] + } + bob: { + id: 2IZNP...UYGYA + exchpub: WFLMZ...B7NHA + signpub: GTGXG...IE3OA + exec: { + sendmail: ["/usr/sbin/sendmail"] + warcer: ["/path/to/warcer.sh"] + wgeter: ["/path/to/wgeter.sh"] + } + freq: { + path: "/home/bob/pub" + chunked: 1024 + minsize: 2048 + } + via: ["alice"] + rxrate: 10 + txrate: 20 + } +} +@end verbatim + +Except for @code{id}, @code{exchpub} and @code{signpub} each neighbour +node has the following fields: + +@table @code + +@item noisepub + If present, then node can be online called using @ref{Sync, + synchronization protocol}. Contains authentication public key. + +@anchor{CfgExec} + @item exec + Dictionary consisting of handles and corresponding command line + arguments. In example above there are @command{sendmail} handles, + @command{warcer}, @command{wgeter} and @command{flag} one. Remote + node can queue some handle execution with providing additional + command line arguments and the body fed to command's @code{stdin}. + + @verb{|sendmail: ["/usr/sbin/sendmail", "-t"]|} handle, when called by + @verb{|echo hello world | nncp-exec self sendmail ARG0 ARG1 ARG2|} + command, will execute: + +@example +NNCP_SELF=OURNODE \ +NNCP_SENDER=REMOTE \ +NNCP_NICE=64 \ +/usr/sbin/sendmail -t ARG0 ARG1 ARG2 +@end example + + feeding @verb{|hello world\n|} to that started @command{sendmail} + process. + +@anchor{CfgIncoming} +@item incoming + Full path to directory where all file uploads will be saved. May be + omitted to forbid file uploading on that node. + +@anchor{CfgFreq} +@item freq + @table @code + @item path + Full path to directory from where file requests will queue files + for transmission. May be omitted to forbid freqing from that node. + + @item chunked + If set, then enable @ref{Chunked, chunked} file transmission + during freqing. This is the desired chunk size in KiBs. + + @item minsize + If set, then apply @ref{OptMinSize, -minsize} option during file + transmission. + @end table + +@anchor{CfgVia} +@item via + An array of node identifiers that will be used as a relay to that + node. For example @verb{|["foo","bar"]|} means that packet can reach + current node by transitioning through @code{foo} and then @code{bar} + nodes. May be omitted if direct connection exists and no relaying is + required. + +@anchor{CfgAddrs} +@item addrs + Dictionary containing known network addresses of the node. Each key + is human-readable name of the address. For direct TCP connections + use @verb{|host:port|} format, pointing to @ref{nncp-daemon}'s + listening instance. + + Also you can pipe connection through the external command using + @verb{#|some command#} format. @code{/bin/sh -c "some command"} will + start and its @code{stdin}/@code{stdout} used as a connection. + + May be omitted if either no direct connection exists, or + @ref{nncp-call} is used with forced address specifying. + +@anchor{CfgXxRate} +@item rxrate/txrate + If greater than zero, then at most *rate packets per second will be + sent/received after the handshake. It could be used as crude + bandwidth traffic shaper: each packet has at most 64 KiB payload + size. If omitted -- no rate limits. + +@anchor{CfgOnlineDeadline} +@item onlinedeadline + Online connection deadline of nodes inactivity in seconds. It is the + time connection considered dead after not receiving/sending any + packets (except for PINGs) and connection must be terminated. By + default it is set to 10 seconds. This can be set to rather high + values to keep connection alive (to reduce handshake overhead and + delays), wait for appearing packets ready to send and notifying + remote side about their appearance. + +@anchor{CfgMaxOnlineTime} +@item maxonlinetime + If greater than zero, then it is maximal time of single connection. + Forcefully disconnect if it is exceeded. + +@anchor{CfgCalls} +@item calls + List of @ref{Call, call configuration}s. + Can be omitted if @ref{nncp-caller} won't be used to call that node. + +@end table diff --git a/doc/cfg/notify.texi b/doc/cfg/notify.texi new file mode 100644 index 0000000..8badc94 --- /dev/null +++ b/doc/cfg/notify.texi @@ -0,0 +1,42 @@ +@node CfgNotify +@section Configuration notification options + +That section controls what notifications are enabled and how must be +sent through the email. Notifications are sent for successful tossing of +file, freq or exec packet. + +@verbatim +notify: { + file: { + from: nncp@localhost + to: user+file@example.com + } + freq: { + from: nncp@localhost + to: user+freq@example.com + } + exec: { + bob.somehandle: { + from: nncp+bob@localhost + to: user+somehandle@example.com + } + *.anotherhandle: { + from: nncp@localhost + to: user+anotherhandle@example.com + } + } +} +@end verbatim + +Corresponding @strong{from} and @strong{to} fields will be substituted +in notification email message. @code{neigh.self.exec.sendmail} will be +used as a local mailer (command called for sending email message). + +You can omit either of those two @code{from}/@code{to} sections to omit +corresponding notifications, or the whole section at once. + +@code{notify.exec} section is a mapping of exec handles and +corresponding @code{from}/@code{to} sections. Each handle has either +@code{NODE.HANDLE} or @code{*.HANDLE} syntax. You can override +notification options for some node with the first type of name. Handle +command's output will be included in notification messages. diff --git a/doc/cfg/self.texi b/doc/cfg/self.texi new file mode 100644 index 0000000..0528b5a --- /dev/null +++ b/doc/cfg/self.texi @@ -0,0 +1,10 @@ +@node CfgSelf +@section Configuration self-node keypairs + +@strong{self} section contains our node's private keypairs. + +@strong{exch*} and @strong{sign*} are used during @ref{Encrypted, +encrypted} packet creation. + +@strong{noise*} are used during @ref{Sync, synchronization protocol} +working in @ref{nncp-call}, @ref{nncp-caller}, @ref{nncp-daemon}. diff --git a/doc/cmd/index.texi b/doc/cmd/index.texi new file mode 100644 index 0000000..cb81543 --- /dev/null +++ b/doc/cmd/index.texi @@ -0,0 +1,104 @@ +@node Commands +@unnumbered Commands + +Nearly all commands have the following common options: + +@table @option +@item -cfg + Path to configuration file. May be overridden by @env{NNCPCFG} + environment variable. If file file is an encrypted @ref{EBlob, + eblob}, then ask for passphrase to decrypt it first. +@item -debug + Print debug messages. Normally this option should not be used. +@item -minsize + @anchor{OptMinSize} + Minimal required resulting packet size, in KiBs. For example if you + send 2 KiB file and set @option{-minsize 4}, then resulting packet + will be 4 KiB (containing file itself and some junk). +@item -nice + Set desired outgoing packet @ref{Niceness, niceness level}. +@item -replynice + Set desired reply packet @ref{Niceness, niceness level}. Only freq + and exec packets look at that niceness level. +@item -via + Override @ref{CfgVia, via} configuration option for destination node. + Specified nodes must be separated with comma: @verb{|NODE1,NODE2|}. + With @verb{|-via -|} you can disable relaying at all. +@item -spool + Override path to spool directory. May be specified by + @env{NNCPSPOOL} environment variable. +@item -log + Override path to logfile. May be specified by @env{NNCPLOG} + environment variable. +@item -quiet + Print only errors, omit simple informational messages. In any case + those messages are logged, so you can reread them using + @ref{nncp-log} command. +@item -progress, -noprogress + Either force progress showing, or disable it. +@item -version + Print version information. +@item -warranty + Print warranty information (no warranty). +@end table + +@menu +Configuration file commands + +* nncp-cfgnew:: +* nncp-cfgmin:: +* nncp-cfgenc:: + +Packets creation commands + +* nncp-file:: +* nncp-exec:: +* nncp-freq:: + +Packets sharing commands + +* nncp-xfer:: +* nncp-bundle:: + +Checking and tossing commands + +* nncp-toss:: +* nncp-check:: +* nncp-reass:: + +Online synchronization protocol commands + +* nncp-daemon:: +* nncp-call:: +* nncp-caller:: +* nncp-cronexpr:: + +Maintenance, monitoring and debugging commands: + +* nncp-stat:: +* nncp-log:: +* nncp-rm:: +* nncp-pkt:: +* nncp-hash:: +@end menu + +@include cmd/nncp-cfgnew.texi +@include cmd/nncp-cfgmin.texi +@include cmd/nncp-cfgenc.texi +@include cmd/nncp-file.texi +@include cmd/nncp-exec.texi +@include cmd/nncp-freq.texi +@include cmd/nncp-xfer.texi +@include cmd/nncp-bundle.texi +@include cmd/nncp-toss.texi +@include cmd/nncp-check.texi +@include cmd/nncp-reass.texi +@include cmd/nncp-daemon.texi +@include cmd/nncp-call.texi +@include cmd/nncp-caller.texi +@include cmd/nncp-cronexpr.texi +@include cmd/nncp-stat.texi +@include cmd/nncp-log.texi +@include cmd/nncp-rm.texi +@include cmd/nncp-pkt.texi +@include cmd/nncp-hash.texi diff --git a/doc/cmd/nncp-bundle.texi b/doc/cmd/nncp-bundle.texi new file mode 100644 index 0000000..9508946 --- /dev/null +++ b/doc/cmd/nncp-bundle.texi @@ -0,0 +1,46 @@ +@node nncp-bundle +@section nncp-bundle + +@example +$ nncp-bundle [options] -tx [-delete] NODE [NODE ...] > ... +$ nncp-bundle [options] -rx -delete [-dryrun] [NODE ...] < ... +$ nncp-bundle [options] -rx [-check] [-dryrun] [NODE ...] < ... +@end example + +With @option{-tx} option, this command creates @ref{Bundles, bundle} of +@ref{Encrypted, encrypted packets} from the spool directory and writes +it to @code{stdout}. + +With @option{-rx} option, this command takes bundle from @code{stdin} +and copies all found packets for our node to the spool directory. Pay +attention that @strong{no} integrity checking is done by default. Modern +tape drives could easily provide too much throughput your CPU won't be +able to verify on the fly. So if you won't @ref{nncp-toss, toss} +received packets at the place, it is advisable either to run +@ref{nncp-check} utility for packets integrity verification, or to use +@option{-check} option to enable on the fly integrity check. + +You can specify multiple @option{NODE} arguments, telling for what nodes +you want to create the stream, or take it from. If no nodes are +specified for @option{-rx} mode, then all packets aimed at us will be +processed. + +When packets are sent through the stream, they are still kept in the +spool directory, because there is no assurance that they are transferred +to the media (media (CD-ROM, tape drive, raw hard drive) can end). If +you want to forcefully delete them (after they are successfully flushed +to @code{stdout}) anyway, use @option{-delete} option. + +But you can verify produced stream after, by digesting it by yourself +with @option{-rx} and @option{-delete} options -- in that mode, stream +packets integrity will be checked and they will be deleted from the +spool if everything is good. So it is advisable to recheck your streams: + +@example +$ nncp-bundle -tx ALICE BOB WHATEVER | cdrecord -tao - +$ dd if=/dev/cd0 bs=2048 | nncp-bundle -rx -delete +@end example + +@option{-dryrun} option prevents any writes to the spool. This is +useful when you need to see what packets will pass by and possibly check +their integrity. diff --git a/doc/cmd/nncp-call.texi b/doc/cmd/nncp-call.texi new file mode 100644 index 0000000..9dc32bd --- /dev/null +++ b/doc/cmd/nncp-call.texi @@ -0,0 +1,60 @@ +@node nncp-call +@section nncp-call + +@example +$ nncp-call [options] + [-onlinedeadline INT] + [-maxonlinetime INT] + [-rx|-tx] + [-list] + [-pkts PKT,PKT,...] + [-rxrate INT] + [-txrate INT] + [-autotoss*] + [-nock] + NODE[:ADDR] [FORCEADDR] +@end example + +Call (connect to) specified @option{NODE} and run @ref{Sync, +synchronization} protocol with the @ref{nncp-daemon, daemon} on the +remote side. Normally this command could be run any time you wish to +either check for incoming packets, or to send out queued ones. +Synchronization protocol allows resuming and bidirectional packets +transfer. + +If @option{-rx} option is specified then only inbound packets +transmission is performed. If @option{-tx} option is specified, then +only outbound transmission is performed. + +@option{-onlinedeadline} overrides @ref{CfgOnlineDeadline, @emph{onlinedeadline}}. +@option{-maxonlinetime} overrides @ref{CfgMaxOnlineTime, @emph{maxonlinetime}}. +@option{-rxrate}/@option{-txrate} override @ref{CfgXxRate, rxrate/txrate}. + +@option{-list} option allows you to list packets of remote node, without +any transmission. You can specify what packets your want to download, by +specifying @option{-pkts} option with comma-separated list of packets +identifiers. + +Each @option{NODE} can contain several uniquely identified +@option{ADDR}esses in @ref{CfgAddrs, configuration} file. If you do +not specify the exact one, then all will be tried until the first +success. Optionally you can force @option{FORCEADDR} address usage, +instead of addresses taken from configuration file. You can specify both +@verb{|host:port|} and @verb{#|some command#} formats. + +@option{-autotoss} option runs tosser on node's spool every second +during the call. All @option{-autotoss-*} options is the same as in +@ref{nncp-toss} command. + +Partly downloaded packets are stored in @file{.part} files. By default +all downloaded files are sequentially checksummed in the background, +stripping @file{.part} extension if is successful. If @option{-nock} +option is set, then no checksumming is done, renaming fully downloaded +files to @file{.nock} extension. Pay attention that checksumming can be +time consuming and connection could be lost during that check, so remote +node won't be notified that the file is finished. If you run +@ref{nncp-check, @command{nncp-check -nock}}, that will checksum files +and strip the @file{.nock} extension, then repeated call to remote node +will notify about packet's completion. Also it will be notified if +@ref{nncp-toss, tossing} created @file{.seen} file. +Read @ref{CfgNoCK, more} about @option{-nock} option. diff --git a/doc/cmd/nncp-caller.texi b/doc/cmd/nncp-caller.texi new file mode 100644 index 0000000..0b309d8 --- /dev/null +++ b/doc/cmd/nncp-caller.texi @@ -0,0 +1,15 @@ +@node nncp-caller +@section nncp-caller + +@example +$ nncp-caller [options] [NODE ...] +@end example + +Croned daemon that calls remote nodes from time to time, according to +their @ref{CfgCalls, @emph{calls}} configuration field. + +Optional number of @option{NODE}s tells to ignore other ones. +Otherwise all nodes with specified @emph{calls} configuration +field will be called. + +Look at @ref{nncp-call} for more information. diff --git a/doc/cmd/nncp-cfgenc.texi b/doc/cmd/nncp-cfgenc.texi new file mode 100644 index 0000000..80fab22 --- /dev/null +++ b/doc/cmd/nncp-cfgenc.texi @@ -0,0 +1,39 @@ +@node nncp-cfgenc +@section nncp-cfgenc + +@example +$ nncp-cfgenc [options] [-s INT] [-t INT] [-p INT] cfg.hjson > cfg.hjson.eblob +$ nncp-cfgenc [options] -d cfg.hjson.eblob > cfg.hjson +@end example + +This command allows you to encrypt provided @file{cfg.hjson} file with +the passphrase, producing @ref{EBlob, eblob}, to safely keep your +configuration file with private keys. This utility was written for users +who do not want (or can not) to use either @url{https://gnupg.org/, +GnuPG} or similar tools. That @file{eblob} file can be used directly in +@option{-cfg} option of nearly all commands. + +@option{-s}, @option{-t}, @option{-p} are used to tune @file{eblob}'s +password strengthening function. Space memory cost (@option{-s}), +specified in number of BLAKE2b-256 blocks (32 bytes), tells how many +memory must be used for hashing -- bigger values are better, but slower. +Time cost (@option{-t}) tells how many rounds/iterations must be +performed -- bigger is better, but slower. Number of parallel jobs +(@option{-p}) tells how many computation processes will be run: this is +the same as running that number of independent hashers and then joining +their result together. + +When invoked for encryption, passphrase is entered manually twice. When +invoked for decryption (@option{-d} option), it is asked once and exits +if passphrase can not decrypt @file{eblob}. + +@option{-dump} options parses @file{eblob} and prints parameters used +during its creation. For example: +@example +$ nncp-cfgenc -dump /usr/local/etc/nncp.hjson.eblob +Strengthening function: Balloon with BLAKE2b-256 +Memory space cost: 1048576 bytes +Number of rounds: 16 +Number of parallel jobs: 2 +Blob size: 2494 +@end example diff --git a/doc/cmd/nncp-cfgmin.texi b/doc/cmd/nncp-cfgmin.texi new file mode 100644 index 0000000..d0125ac --- /dev/null +++ b/doc/cmd/nncp-cfgmin.texi @@ -0,0 +1,11 @@ +@node nncp-cfgmin +@section nncp-cfgmin + +@example +$ nncp-cfgmin [options] > stripped.hjson +@end example + +Print out stripped configuration version: only path to @ref{Spool, +spool}, path to log file, neighbours public keys are stayed. This is +useful mainly for usage with @ref{nncp-xfer} that has to know only +neighbours, without private keys involving. diff --git a/doc/cmd/nncp-cfgnew.texi b/doc/cmd/nncp-cfgnew.texi new file mode 100644 index 0000000..de3c71a --- /dev/null +++ b/doc/cmd/nncp-cfgnew.texi @@ -0,0 +1,17 @@ +@node nncp-cfgnew +@section nncp-cfgnew + +@example +$ nncp-cfgnew [options] [-area NAME] [-nocomments] > new.hjson +@end example + +Generate new node configuration: private keys, example configuration +file and print it to @code{stdout}. You must use this command when you +setup the new node. @option{-nocomments} will create configuration file +without descriptive huge comments -- useful for advanced users. + +With @option{-area} option you generate only the @ref{Area, area} +related part of the configuration file. + +Pay attention that private keys generation consumes an entropy from your +operating system. diff --git a/doc/cmd/nncp-check.texi b/doc/cmd/nncp-check.texi new file mode 100644 index 0000000..4a257af --- /dev/null +++ b/doc/cmd/nncp-check.texi @@ -0,0 +1,18 @@ +@node nncp-check +@section nncp-check + +@example +$ nncp-check [-nock] [-cycle INT] [options] +@end example + +Perform @ref{Spool, spool} directory integrity check. Read all files +that has Base32-encoded filenames and compare it with recalculated +@ref{MTH} hash output of their contents. + +The most useful mode of operation is with @option{-nock} option, that +checks integrity of @file{.nock} files, renaming them to ordinary +(verified) encrypted packets. + +@option{-cycle} option tells not to quit, but to repeat checking every +@option{INT} seconds in an infinite loop. That can be useful when +running this command as a daemon. diff --git a/doc/cmd/nncp-cronexpr.texi b/doc/cmd/nncp-cronexpr.texi new file mode 100644 index 0000000..f06202a --- /dev/null +++ b/doc/cmd/nncp-cronexpr.texi @@ -0,0 +1,24 @@ +@node nncp-cronexpr +@section nncp-cronexpr + +@example +$ nncp-cronexpr -num 12 "*/1 * * * * SAT,SUN 2021" +@end example + +Check validity of specified @ref{CronExpr, cron expression} and print 12 +next time entities: + +@example +$ nncp-cronexpr "*/5 * * * * * *" +Now: 2021-07-04T08:26:26.229285858Z +0: 2021-07-04T08:26:30Z +1: 2021-07-04T08:26:35Z +2: 2021-07-04T08:26:40Z +3: 2021-07-04T08:26:45Z +4: 2021-07-04T08:26:50Z +5: 2021-07-04T08:26:55Z +6: 2021-07-04T08:27:00Z +7: 2021-07-04T08:27:05Z +8: 2021-07-04T08:27:10Z +9: 2021-07-04T08:27:15Z +@end example diff --git a/doc/cmd/nncp-daemon.texi b/doc/cmd/nncp-daemon.texi new file mode 100644 index 0000000..aa11aca --- /dev/null +++ b/doc/cmd/nncp-daemon.texi @@ -0,0 +1,36 @@ +@node nncp-daemon +@section nncp-daemon + +@example +$ nncp-daemon [options] + [-maxconn INT] [-bind ADDR] [-inetd] + [-autotoss*] [-nock] [-mcd-once] +@end example + +Start listening TCP daemon, wait for incoming connections and run +@ref{Sync, synchronization protocol} with each of them. You can run +@ref{nncp-toss} utility in background to process inbound packets from +time to time. + +@option{-maxconn} option specifies how many simultaneous clients daemon +can handle. @option{-bind} option specifies @option{addr:port} it must +bind to and listen. + +It could be run as @command{inetd} service, by specifying +@option{-inetd} option. Pay attention that because it uses +@code{stdin}/@code{stdout}, it can not effectively work with IO timeouts +and connection closing can propagate up to 5 minutes in practice. +Example inetd-entry: + +@verbatim +uucp stream tcp6 nowait nncpuser /usr/local/bin/nncp-daemon nncp-daemon -quiet -inetd +@end verbatim + +@option{-autotoss} option runs tosser on node's spool every second +during the call. All @option{-autotoss-*} options is the same as in +@ref{nncp-toss} command. + +Read @ref{CfgNoCK, more} about @option{-nock} option. + +@option{-mcd-once} option sends @ref{MCD} announcements once and quits. +Could be useful with inetd-based setup, where daemons are not running. diff --git a/doc/cmd/nncp-exec.texi b/doc/cmd/nncp-exec.texi new file mode 100644 index 0000000..7bd7508 --- /dev/null +++ b/doc/cmd/nncp-exec.texi @@ -0,0 +1,51 @@ +@node nncp-exec +@section nncp-exec + +@example +$ nncp-exec [options] [-use-tmp] [-nocompress] NODE HANDLE [ARG0 ARG1 ...] +$ nncp-exec [options] [-use-tmp] [-nocompress] area:AREA HANDLE [ARG0 ARG1 ...] +@end example + +Send execution command to @option{NODE} for specified @option{HANDLE}. +Body is read from @code{stdin} into memory and compressed (unless +@option{-nocompress} is specified). After receiving, remote side will +execute specified @ref{CfgExec, handle} command with @option{ARG*} +appended and decompressed body fed to command's @code{stdin}. + +If @option{-use-tmp} option is specified, then @code{stdin} data is read +into temporary file first, requiring twice more disk space, but no +memory requirements. @ref{StdinTmpFile, Same temporary file} rules +applies as with @ref{nncp-file, nncp-file -} command. + +For example, if remote side has following configuration file for your +node: + +@verbatim +exec: { + sendmail: [/usr/sbin/sendmail, "-t"] + appender: ["/bin/sh", "-c", "cat >> /append"] +} +@end verbatim + +then executing @verb{|echo My message | nncp-exec -replynice 123 REMOTE +sendmail root@localhost|} will lead to execution of: + +@example +echo My message | + NNCP_SELF=REMOTE \ + NNCP_SENDER=OurNodeId \ + NNCP_NICE=123 \ + /usr/sbin/sendmail -t root@@localhost +@end example + +If @ref{CfgNotify, notification} is enabled on the remote side for exec +handles, then it will sent simple letter after successful command +execution with its output in message body. + +@strong{Pay attention} that packet generated with this command won't be +be chunked. + +If you use @option{area:AREA} instead of @option{NODE}, then +@ref{Multicast, multicast} packet will be sent to specified area. That +creates outgoing packet to the @strong{self} node, so you have to run +@ref{nncp-toss, tossing} to create outgoing packets to required subscribers. diff --git a/doc/cmd/nncp-file.texi b/doc/cmd/nncp-file.texi new file mode 100644 index 0000000..29f061a --- /dev/null +++ b/doc/cmd/nncp-file.texi @@ -0,0 +1,56 @@ +@node nncp-file +@section nncp-file + +@example +$ nncp-file [options] [-chunked INT] SRC NODE:[DST] +$ nncp-file [options] [-chunked INT] SRC area:AREA:[DST] +@end example + +Send @file{SRC} file to remote @option{NODE}. @file{DST} specifies +destination file name in remote's @ref{CfgIncoming, incoming} +directory. If this file already exists there, then counter will be +appended to it. + +This command queues file in @ref{Spool, spool} directory immediately +(through the temporary file of course) -- so pay attention that sending +2 GiB file will create 2 GiB outbound encrypted packet. + +@anchor{StdinTmpFile} +If @file{SRC} equals to @file{-}, then create an encrypted temporary +file and copy everything taken from @code{stdin} to it and use for outbound +packet creation. Pay attention that if you want to send 1 GiB of data +taken from @code{stdin}, then you have to have more than 2 GiB of disk space +for that temporary file and resulting encrypted packet. You can control +temporary file location directory with @env{TMPDIR} environment +variable. Encryption is performed in AEAD mode with +@url{https://cr.yp.to/chacha.html, ChaCha20}-@url{https://en.wikipedia.org/wiki/Poly1305, Poly1305} +algorithms. Data is divided on 128 KiB blocks. Each block is encrypted +with increasing nonce counter. File is deletes immediately after +creation, so even if program crashes -- disk space will be reclaimed, no +need in cleaning it up later. + +If @file{SRC} points to directory, then +@url{https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_01, pax archive} +will be created on the fly with directory contents and destination +filename @file{.tar} appended. It @strong{won't} contain any entities +metainformation, but modification time with the names. UID/GID are set +to zero. Directories have 777 permissions, files have 666, for being +friendly with @command{umask}. Also each entity will have comment like +@verb{|Autogenerated by NNCP version X.Y.Z built with goXXX|}. + +If @option{-chunked} is specified, then source file will be split +@ref{Chunked, on chunks}. @option{INT} is the desired chunk size in +KiBs. This mode is more CPU hungry. Pay attention that chunk is saved in +spool directory immediately and it is not deleted if any error occurs. +@option{-minsize} option is applied per each chunk. Do not forget about +@ref{ChunkedZFS, possible} ZFS deduplication issues. Zero +@option{-chunked} disables chunked transmission. + +If @ref{CfgNotify, notification} is enabled on the remote side for +file transmissions, then it will sent simple letter after successful +file receiving. + +If you use @option{area:AREA} instead of @option{NODE}, then +@ref{Multicast, multicast} packet will be sent to specified area. That +creates outgoing packet to the @strong{self} node, so you have to run +@ref{nncp-toss, tossing} to create outgoing packets to required subscribers. diff --git a/doc/cmd/nncp-freq.texi b/doc/cmd/nncp-freq.texi new file mode 100644 index 0000000..aa35f99 --- /dev/null +++ b/doc/cmd/nncp-freq.texi @@ -0,0 +1,15 @@ +@node nncp-freq +@section nncp-freq + +@example +$ nncp-freq [options] NODE:SRC [DST] +@end example + +Send file request to @option{NODE}, asking it to send its @file{SRC} +file from @ref{CfgFreq, freq.path} directory to our node under @file{DST} +filename in our @ref{CfgIncoming, incoming} one. If @file{DST} is not +specified, then last element of @file{SRC} will be used. + +If @ref{CfgNotify, notification} is enabled on the remote side for +file request, then it will sent simple letter after successful file +queuing. diff --git a/doc/cmd/nncp-hash.texi b/doc/cmd/nncp-hash.texi new file mode 100644 index 0000000..adc6e8f --- /dev/null +++ b/doc/cmd/nncp-hash.texi @@ -0,0 +1,17 @@ +@node nncp-hash +@section nncp-hash + +@example +$ nncp-hash [-file ...] [-seek X] [-debug] [-progress] +@end example + +Calculate @ref{MTH} hash of either stdin, or @option{-file} if +specified. + +You can optionally force seeking the file first, reading only part of +the file, and then prepending unread portion of data, with the +@option{-seek} option. It is intended only for testing and debugging of +MTH hasher capabilities. + +@option{-debug} option shows all intermediate MTH hashes. +And @option{-progress} will show progress bar. diff --git a/doc/cmd/nncp-log.texi b/doc/cmd/nncp-log.texi new file mode 100644 index 0000000..6ac6a7a --- /dev/null +++ b/doc/cmd/nncp-log.texi @@ -0,0 +1,9 @@ +@node nncp-log +@section nncp-log + +@example +$ nncp-log [options] +@end example + +Parse @ref{Log, log} file and print out its records in short +human-readable form. diff --git a/doc/cmd/nncp-pkt.texi b/doc/cmd/nncp-pkt.texi new file mode 100644 index 0000000..b3391b4 --- /dev/null +++ b/doc/cmd/nncp-pkt.texi @@ -0,0 +1,44 @@ +@node nncp-pkt +@section nncp-pkt + +@example +$ nncp-pkt [options] < pkt +$ nncp-pkt [options] [-decompress] -dump < pkt > payload +$ nncp-pkt -overheads +@end example + +Low level packet parser. Can be useful for debugging. There are two +types of packets: @ref{Plain, plain} and @ref{Encrypted, encrypted}. By +default it will print packet's header, for example: + +@example +Packet type: encrypted +Niceness: B (224) +Sender: 2WHBV3TPZHDOZGUJEH563ZEK7M33J4UESRFO4PDKWD5KZNPROABQ (self) +@end example + +@option{-dump} option outputs plain packet's payload (if it is file +transmission, then it will be the file itself as an example). If it is +an encrypted packet, then it will be decrypted first, outputing the +included plain packet, that can be fed to @command{nncp-pkt} again: + +@example +Packet type: plain +Payload type: transitional +Niceness: B (224) +Path: VHMTRWDOXPLK7BR55ICZ5N32ZJUMRKZEMFNGGCEAXV66GG43PEBQ (name-of-node) + +Packet type: plain +Payload type: exec compressed +Niceness: P (96) +Path: stargrave@@stargrave.org +@end example + +@option{-decompress} option tries to zstd-decompress data from plain +packet (useful with @verb{|exec compressed|} types of packets). + +@option{-overheads} options print encrypted, plain and size header overheads. + +This command automatically determines if an encrypted packet belongs to +@ref{Multicast, multicast} area and will try to decrypt it with its +corresponding key. diff --git a/doc/cmd/nncp-reass.texi b/doc/cmd/nncp-reass.texi new file mode 100644 index 0000000..7cae84b --- /dev/null +++ b/doc/cmd/nncp-reass.texi @@ -0,0 +1,58 @@ +@node nncp-reass +@section nncp-reass + +@example +$ nncp-reass [options] [-dryrun] [-keep] [-dump] [-stdout] FILE.nncp.meta +$ nncp-reass [options] [-dryrun] [-keep] @{-all | -node NODE@} +@end example + +Reassemble @ref{Chunked, chunked file} after @ref{nncp-toss, tossing}. + +When called with @option{FILE} option, this command will reassemble only +it. When called with @option{-node} option, this command will try to +reassemble all @file{.nncp.meta} files found in @option{NODE}'s +@ref{CfgIncoming, incoming} directory. When called with @option{-all} +option, then cycle through all known nodes to do the same. + +Reassembling process does the following: + +@enumerate +@item Parses @ref{Chunked, @file{.nncp.meta}} file. +@item Checks existence and size of every @file{.nncp.chunkXXX}. +@item Verifies integrity of every chunk. +@item Concatenates all chunks, simultaneously removing them from filesystem. +@end enumerate + +That process reads the whole data twice. Be sure to have free disk +space for at least one chunk. Decrypted chunk files as a rule are saved +in pseudo-random order, so removing them during reassembly process will +likely lead to filesystem fragmentation. Reassembly process on +filesystems with deduplication capability should be rather lightweight. + +If @option{-dryrun} option is specified, then only existence and +integrity checking are performed. + +If @option{-keep} option is specified, then no +@file{.nncp.meta}/@file{.nncp.chunkXXX} files are deleted during +reassembly process. + +@option{-stdout} option outputs reassembled file to @code{stdout}, +instead of saving to temporary file with renaming after. This could be +useful for reassembling on separate filesystem to lower fragmentation +effect, and/or separate storage device for higher performance. + +@option{-dump} option prints meta-file contents in human-friendly form. +It is useful mainly for debugging purposes. For example: +@example +Original filename: testfile +File size: 3.8 MiB (3987795 bytes) +Chunk size: 1.0 MiB (1048576 bytes) +Number of chunks: 4 +Checksums: + 0: eac60d819edf40b8ecdacd0b9a5a8c62de2d15eef3c8ca719eafa0be9b894017 + 1: 013a07e659f2e353d0e4339c3375c96c7fffaa2fa00875635f440bbc4631052a + 2: f4f883975a663f2252328707a30e71b2678f933b2f3103db8475b03293e4316e + 3: 0e9e229501bf0ca42d4aa07393d19406d40b179f3922a3986ef12b41019b45a3 +@end example + +Do not forget about @ref{ChunkedZFS, possible} ZFS deduplication issues. diff --git a/doc/cmd/nncp-rm.texi b/doc/cmd/nncp-rm.texi new file mode 100644 index 0000000..8d2db29 --- /dev/null +++ b/doc/cmd/nncp-rm.texi @@ -0,0 +1,53 @@ +@node nncp-rm +@section nncp-rm + +@example +$ nncp-rm [options] -tmp +$ nncp-rm [options] -lock +$ nncp-rm [options] @{-all|-node NODE@} -part +$ nncp-rm [options] @{-all|-node NODE@} -seen +$ nncp-rm [options] @{-all|-node NODE@} -nock +$ nncp-rm [options] @{-all|-node NODE@} -hdr +$ nncp-rm [options] @{-all|-node NODE@} -area +$ nncp-rm [options] @{-all|-node NODE@} [-rx] [-tx] +$ nncp-rm [options] @{-all|-node NODE@} -pkt PKT +@end example + +This command is aimed to delete various files from your spool directory: + +@itemize + +@item If @option{-tmp} option is specified, then it will delete all +temporary files in @file{spool/tmp} directory. Files may stay in it when +commands like @ref{nncp-file} fail for some reason. + +@item If @option{-lock} option is specified, then all @file{.lock} files +will be deleted in your spool directory. + +@item If @option{-pkt} option is specified, then @file{PKT} packet (its +Base32 name) will be deleted. This is useful when you see some packet +failing to be processed. + +@item When either @option{-rx} or @option{-tx} options are specified +(maybe both of them), then delete all packets from that given queues. + +@item @option{-part} option deletes @file{.part}ly downloaded files. + +@item @option{-seen} option deletes @file{.seen} files. But it does not +apply to @ref{Multicast, multicast areas} @file{.seen} ones! + +@item @option{-nock} option deletes non-checksummed (non-verified) +@file{.nock} files. + +@item @option{-hdr} option deletes cached @file{.hdr} files. + +@item @option{-area} option deletes @file{.seen} files in @file{area/} +subdirectories. + +@end itemize + +@option{-dryrun} option just prints what will be deleted. + +You can also select files that only have modification date older than specified +@option{-older} time units (@code{10s} (10 seconds), @code{5m} (5 minutes), +@code{12h} (12 hours), @code{2d} (2 days)). diff --git a/doc/cmd/nncp-stat.texi b/doc/cmd/nncp-stat.texi new file mode 100644 index 0000000..5759a01 --- /dev/null +++ b/doc/cmd/nncp-stat.texi @@ -0,0 +1,13 @@ +@node nncp-stat +@section nncp-stat + +@example +$ nncp-stat [options] [-pkt] [-node NODE] +@end example + +Print current @ref{Spool, spool} statistics about unsent and unprocessed +packets. For each node (unless @option{-node} specified) and each +niceness level there will be printed how many packets (with the total +size) are in inbound (Rx) and outbound (Tx) queues, how many +unchecksummed @file{.nock} packets or partly downloaded @file{.part} +ones. @option{-pkt} option show information about each packet. diff --git a/doc/cmd/nncp-toss.texi b/doc/cmd/nncp-toss.texi new file mode 100644 index 0000000..703c291 --- /dev/null +++ b/doc/cmd/nncp-toss.texi @@ -0,0 +1,32 @@ +@node nncp-toss +@section nncp-toss + +@example +$ nncp-toss [options] + [-node NODE] + [-dryrun] + [-cycle INT] + [-seen] + [-nofile] [-nofreq] [-noexec] [-notrns] [-noarea] +@end example + +Perform "tossing" operation on all inbound packets. This is the tool +that decrypts all packets and processes all payload packets in them: +copies files, sends mails, sends out file requests and relays transition +packets. It should be run after each online/offline exchange. + +@option{-dryrun} option does not perform any writing and sending, just +tells what it will do. + +@option{-cycle} option tells not to quit, but to repeat tossing every +@option{INT} seconds in an infinite loop. That can be useful when +running this command as a daemon. + +@option{-seen} option creates empty @file{XXX.seen} file after +successful tossing of @file{XXX} packet. @ref{nncp-xfer}, +@ref{nncp-bundle}, @ref{nncp-daemon} and @ref{nncp-call} commands skip +inbound packets that has been already seen, processed and tossed. This +is helpful to prevent duplicates. + +@option{-nofile}, @option{-nofreq}, @option{-noexec}, @option{-notrns}, +@option{-noarea} options allow to disable any kind of packet types processing. diff --git a/doc/cmd/nncp-xfer.texi b/doc/cmd/nncp-xfer.texi new file mode 100644 index 0000000..70edaf8 --- /dev/null +++ b/doc/cmd/nncp-xfer.texi @@ -0,0 +1,31 @@ +@node nncp-xfer +@section nncp-xfer + +@example +$ nncp-xfer [options] [-node NODE] [-mkdir] [-keep] [-rx|-tx] DIR +@end example + +Search for directory in @file{DIR} containing inbound packets for us and +move them to local @ref{Spool, spool} directory. Also search for known +neighbours directories and move locally queued outbound packets to them. +This command is used for offline packets transmission. + +If @option{-mkdir} option is specified, then outbound neighbour(s) +directories will be created. This is useful for the first time usage, +when storage device does not have any directories tree. + +If @option{-keep} option is specified, then keep copied files, do not +remove them. + +@option{-rx} option tells only to move inbound packets addressed to us. +@option{-tx} option tells exactly the opposite: move only outbound packets. + +@ref{nncp-cfgmin} could be useful for creating stripped minimalistic +configuration file version without any private keys. + +@file{DIR} directory has the following structure: +@file{RECIPIENT/SENDER/PACKET}, where @file{RECIPIENT} is Base32 encoded +destination node, @file{SENDER} is Base32 encoded sender node. + +Also look for @ref{nncp-bundle}, especially if you deal with CD-ROM and +tape drives. diff --git a/doc/cmds.texi b/doc/cmds.texi deleted file mode 100644 index 665b576..0000000 --- a/doc/cmds.texi +++ /dev/null @@ -1,656 +0,0 @@ -@node Commands -@unnumbered Commands - -Nearly all commands have the following common options: - -@table @option -@item -cfg - Path to configuration file. May be overridden by @env{NNCPCFG} - environment variable. If file file is an encrypted @ref{EBlob, - eblob}, then ask for passphrase to decrypt it first. -@item -debug - Print debug messages. Normally this option should not be used. -@item -minsize - @anchor{OptMinSize} - Minimal required resulting packet size, in KiBs. For example if you - send 2 KiB file and set @option{-minsize 4}, then resulting packet - will be 4 KiB (containing file itself and some junk). -@item -nice - Set desired outgoing packet @ref{Niceness, niceness level}. -@item -replynice - Set desired reply packet @ref{Niceness, niceness level}. Only freq - and exec packets look at that niceness level. -@item -via - Override @ref{CfgVia, via} configuration option for destination node. - Specified nodes must be separated with comma: @verb{|NODE1,NODE2|}. - With @verb{|-via -|} you can disable relaying at all. -@item -spool - Override path to spool directory. May be specified by - @env{NNCPSPOOL} environment variable. -@item -log - Override path to logfile. May be specified by @env{NNCPLOG} - environment variable. -@item -quiet - Print only errors, omit simple informational messages. In any case - those messages are logged, so you can reread them using - @ref{nncp-log} command. -@item -progress, -noprogress - Either force progress showing, or disable it. -@item -version - Print version information. -@item -warranty - Print warranty information (no warranty). -@end table - -@node nncp-bundle -@section nncp-bundle - -@example -$ nncp-bundle [options] -tx [-delete] NODE [NODE ...] > ... -$ nncp-bundle [options] -rx -delete [-dryrun] [NODE ...] < ... -$ nncp-bundle [options] -rx [-check] [-dryrun] [NODE ...] < ... -@end example - -With @option{-tx} option, this command creates @ref{Bundles, bundle} of -@ref{Encrypted, encrypted packets} from the spool directory and writes -it to @code{stdout}. - -With @option{-rx} option, this command takes bundle from @code{stdin} -and copies all found packets for our node to the spool directory. Pay -attention that @strong{no} integrity checking is done by default. Modern -tape drives could easily provide too much throughput your CPU won't be -able to verify on the fly. So if you won't @ref{nncp-toss, toss} -received packets at the place, it is advisable either to run -@ref{nncp-check} utility for packets integrity verification, or to use -@option{-check} option to enable on the fly integrity check. - -You can specify multiple @option{NODE} arguments, telling for what nodes -you want to create the stream, or take it from. If no nodes are -specified for @option{-rx} mode, then all packets aimed at us will be -processed. - -When packets are sent through the stream, they are still kept in the -spool directory, because there is no assurance that they are transferred -to the media (media (CD-ROM, tape drive, raw hard drive) can end). If -you want to forcefully delete them (after they are successfully flushed -to @code{stdout}) anyway, use @option{-delete} option. - -But you can verify produced stream after, by digesting it by yourself -with @option{-rx} and @option{-delete} options -- in that mode, stream -packets integrity will be checked and they will be deleted from the -spool if everything is good. So it is advisable to recheck your streams: - -@example -$ nncp-bundle -tx ALICE BOB WHATEVER | cdrecord -tao - -$ dd if=/dev/cd0 bs=2048 | nncp-bundle -rx -delete -@end example - -@option{-dryrun} option prevents any writes to the spool. This is -useful when you need to see what packets will pass by and possibly check -their integrity. - -@node nncp-call -@section nncp-call - -@example -$ nncp-call [options] - [-onlinedeadline INT] - [-maxonlinetime INT] - [-rx|-tx] - [-list] - [-pkts PKT,PKT,...] - [-rxrate INT] - [-txrate INT] - [-autotoss*] - [-nock] - NODE[:ADDR] [FORCEADDR] -@end example - -Call (connect to) specified @option{NODE} and run @ref{Sync, -synchronization} protocol with the @ref{nncp-daemon, daemon} on the -remote side. Normally this command could be run any time you wish to -either check for incoming packets, or to send out queued ones. -Synchronization protocol allows resuming and bidirectional packets -transfer. - -If @option{-rx} option is specified then only inbound packets -transmission is performed. If @option{-tx} option is specified, then -only outbound transmission is performed. - -@option{-onlinedeadline} overrides @ref{CfgOnlineDeadline, @emph{onlinedeadline}}. -@option{-maxonlinetime} overrides @ref{CfgMaxOnlineTime, @emph{maxonlinetime}}. -@option{-rxrate}/@option{-txrate} override @ref{CfgXxRate, rxrate/txrate}. -Read @ref{CfgNoCK, more} about @option{-nock} option. - -@option{-list} option allows you to list packets of remote node, without -any transmission. You can specify what packets your want to download, by -specifying @option{-pkts} option with comma-separated list of packets -identifiers. - -Each @option{NODE} can contain several uniquely identified -@option{ADDR}esses in @ref{CfgAddrs, configuration} file. If you do -not specify the exact one, then all will be tried until the first -success. Optionally you can force @option{FORCEADDR} address usage, -instead of addresses taken from configuration file. You can specify both -@verb{|host:port|} and @verb{#|some command#} formats. - -Pay attention that this command runs integrity check for each completely -received packet in the background. This can be time consuming. -Connection could be lost during that check and remote node won't be -notified that file is done. But after successful integrity check that -file is renamed from @file{.part} one and when you rerun -@command{nncp-call} again, remote node will receive completion -notification. - -@option{-autotoss} option runs tosser on node's spool every second -during the call. All @option{-autotoss-*} options is the same as in -@ref{nncp-toss} command. - -@node nncp-caller -@section nncp-caller - -@example -$ nncp-caller [options] [NODE ...] -@end example - -Croned daemon that calls remote nodes from time to time, according to -their @ref{CfgCalls, @emph{calls}} configuration field. - -Optional number of @option{NODE}s tells to ignore other ones. -Otherwise all nodes with specified @emph{calls} configuration -field will be called. - -Look at @ref{nncp-call} for more information. - -@node nncp-cfgenc -@section nncp-cfgenc - -@example -$ nncp-cfgenc [options] [-s INT] [-t INT] [-p INT] cfg.hjson > cfg.hjson.eblob -$ nncp-cfgenc [options] -d cfg.hjson.eblob > cfg.hjson -@end example - -This command allows you to encrypt provided @file{cfg.hjson} file with -the passphrase, producing @ref{EBlob, eblob}, to safely keep your -configuration file with private keys. This utility was written for users -who do not want (or can not) to use either @url{https://gnupg.org/, -GnuPG} or similar tools. That @file{eblob} file can be used directly in -@option{-cfg} option of nearly all commands. - -@option{-s}, @option{-t}, @option{-p} are used to tune @file{eblob}'s -password strengthening function. Space memory cost (@option{-s}), -specified in number of BLAKE2b-256 blocks (32 bytes), tells how many -memory must be used for hashing -- bigger values are better, but slower. -Time cost (@option{-t}) tells how many rounds/iterations must be -performed -- bigger is better, but slower. Number of parallel jobs -(@option{-p}) tells how many computation processes will be run: this is -the same as running that number of independent hashers and then joining -their result together. - -When invoked for encryption, passphrase is entered manually twice. When -invoked for decryption (@option{-d} option), it is asked once and exits -if passphrase can not decrypt @file{eblob}. - -@option{-dump} options parses @file{eblob} and prints parameters used -during its creation. For example: -@example -$ nncp-cfgenc -dump /usr/local/etc/nncp.hjson.eblob -Strengthening function: Balloon with BLAKE2b-256 -Memory space cost: 1048576 bytes -Number of rounds: 16 -Number of parallel jobs: 2 -Blob size: 2494 -@end example - -@node nncp-cfgmin -@section nncp-cfgmin - -@example -$ nncp-cfgmin [options] > stripped.hjson -@end example - -Print out stripped configuration version: only path to @ref{Spool, -spool}, path to log file, neighbours public keys are stayed. This is -useful mainly for usage with @ref{nncp-xfer} that has to know only -neighbours, without private keys involving. - -@node nncp-cfgnew -@section nncp-cfgnew - -@example -$ nncp-cfgnew [options] [-nocomments] > new.hjson -@end example - -Generate new node configuration: private keys, example configuration -file and print it to @code{stdout}. You must use this command when you -setup the new node. @option{-nocomments} will create configuration file -without descriptive huge comments -- useful for advanced users. - -Pay attention that private keys generation consumes an entropy from your -operating system. - -@node nncp-check -@section nncp-check - -@example -$ nncp-check [-nock] [options] -@end example - -Perform @ref{Spool, spool} directory integrity check. Read all files -that has Base32-encoded filenames and compare it with recalculated -@ref{MTH} hash output of their contents. - -The most useful mode of operation is with @option{-nock} option, that -checks integrity of @file{.nock} files, renaming them to ordinary -(verified) encrypted packets. - -@node nncp-cronexpr -@section nncp-cronexpr - -@example -$ nncp-cronexpr -num 12 "*/1 * * * * SAT,SUN 2021" -@end example - -Check validity of specified @ref{CronExpr, cron expression} and print 12 -next time entities. - -@node nncp-daemon -@section nncp-daemon - -@example -$ nncp-daemon [options] - [-maxconn INT] [-bind ADDR] [-inetd] - [-autotoss*] [-nock] [-mcd-once] -@end example - -Start listening TCP daemon, wait for incoming connections and run -@ref{Sync, synchronization protocol} with each of them. You can run -@ref{nncp-toss} utility in background to process inbound packets from -time to time. - -@option{-maxconn} option specifies how many simultaneous clients daemon -can handle. @option{-bind} option specifies @option{addr:port} it must -bind to and listen. - -It could be run as @command{inetd} service, by specifying -@option{-inetd} option. Pay attention that because it uses -@code{stdin}/@code{stdout}, it can not effectively work with IO timeouts -and connection closing can propagate up to 5 minutes in practice. -Example inetd-entry: - -@verbatim -uucp stream tcp6 nowait nncpuser /usr/local/bin/nncp-daemon nncp-daemon -quiet -inetd -@end verbatim - -@option{-autotoss} option runs tosser on node's spool every second -during the call. All @option{-autotoss-*} options is the same as in -@ref{nncp-toss} command. - -Read @ref{CfgNoCK, more} about @option{-nock} option. - -@option{-mcd-once} option sends @ref{MCD} announcements once and quits. -Could be useful with inetd-based setup, where daemons are not running. - -@node nncp-exec -@section nncp-exec - -@example -$ nncp-exec [options] [-use-tmp] [-nocompress] NODE HANDLE [ARG0 ARG1 ...] -@end example - -Send execution command to @option{NODE} for specified @option{HANDLE}. -Body is read from @code{stdin} into memory and compressed (unless -@option{-nocompress} is specified). After receiving, remote side will -execute specified @ref{CfgExec, handle} command with @option{ARG*} -appended and decompressed body fed to command's @code{stdin}. - -If @option{-use-tmp} option is specified, then @code{stdin} data is read -into temporary file first, requiring twice more disk space, but no -memory requirements. @ref{StdinTmpFile, Same temporary file} rules -applies as with @ref{nncp-file, nncp-file -} command. - -For example, if remote side has following configuration file for your -node: - -@verbatim -exec: { - sendmail: [/usr/sbin/sendmail, "-t"] - appender: ["/bin/sh", "-c", "cat >> /append"] -} -@end verbatim - -then executing @verb{|echo My message | nncp-exec -replynice 123 REMOTE -sendmail root@localhost|} will lead to execution of: - -@example -echo My message | - NNCP_SELF=REMOTE \ - NNCP_SENDER=OurNodeId \ - NNCP_NICE=123 \ - /usr/sbin/sendmail -t root@@localhost -@end example - -If @ref{CfgNotify, notification} is enabled on the remote side for exec -handles, then it will sent simple letter after successful command -execution with its output in message body. - -@strong{Pay attention} that packet generated with this command won't be -be chunked. - -@node nncp-file -@section nncp-file - -@example -$ nncp-file [options] [-chunked INT] SRC NODE:[DST] -@end example - -Send @file{SRC} file to remote @option{NODE}. @file{DST} specifies -destination file name in remote's @ref{CfgIncoming, incoming} -directory. If this file already exists there, then counter will be -appended to it. - -This command queues file in @ref{Spool, spool} directory immediately -(through the temporary file of course) -- so pay attention that sending -2 GiB file will create 2 GiB outbound encrypted packet. - -@anchor{StdinTmpFile} -If @file{SRC} equals to @file{-}, then create an encrypted temporary -file and copy everything taken from @code{stdin} to it and use for outbound -packet creation. Pay attention that if you want to send 1 GiB of data -taken from @code{stdin}, then you have to have more than 2 GiB of disk space -for that temporary file and resulting encrypted packet. You can control -temporary file location directory with @env{TMPDIR} environment -variable. Encryption is performed in AEAD mode with -@url{https://cr.yp.to/chacha.html, ChaCha20}-@url{https://en.wikipedia.org/wiki/Poly1305, Poly1305} -algorithms. Data is divided on 128 KiB blocks. Each block is encrypted -with increasing nonce counter. File is deletes immediately after -creation, so even if program crashes -- disk space will be reclaimed, no -need in cleaning it up later. - -If @file{SRC} points to directory, then -@url{https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_01, pax archive} -will be created on the fly with directory contents and destination -filename @file{.tar} appended. It @strong{won't} contain any entities -metainformation, but modification time with the names. UID/GID are set -to zero. Directories have 777 permissions, files have 666, for being -friendly with @command{umask}. Also each entity will have comment like -@verb{|Autogenerated by NNCP version X.Y.Z built with goXXX|}. - -If @option{-chunked} is specified, then source file will be split -@ref{Chunked, on chunks}. @option{INT} is the desired chunk size in -KiBs. This mode is more CPU hungry. Pay attention that chunk is saved in -spool directory immediately and it is not deleted if any error occurs. -@option{-minsize} option is applied per each chunk. Do not forget about -@ref{ChunkedZFS, possible} ZFS deduplication issues. Zero -@option{-chunked} disables chunked transmission. - -If @ref{CfgNotify, notification} is enabled on the remote side for -file transmissions, then it will sent simple letter after successful -file receiving. - -@node nncp-freq -@section nncp-freq - -@example -$ nncp-freq [options] NODE:SRC [DST] -@end example - -Send file request to @option{NODE}, asking it to send its @file{SRC} -file from @ref{CfgFreq, freq.path} directory to our node under @file{DST} -filename in our @ref{CfgIncoming, incoming} one. If @file{DST} is not -specified, then last element of @file{SRC} will be used. - -If @ref{CfgNotify, notification} is enabled on the remote side for -file request, then it will sent simple letter after successful file -queuing. - -@node nncp-hash -@section nncp-hash - -@example -$ nncp-log [-file ...] [-seek X] [-debug] [-progress] -@end example - -Calculate @ref{MTH} hash of either stdin, or @option{-file} if -specified. - -You can optionally force seeking the file first, reading only part of -the file, and then prepending unread portion of data, with the -@option{-seek} option. It is intended only for testing and debugging of -MTH hasher capabilities. - -@option{-debug} option shows all intermediate MTH hashes. -And @option{-progress} will show progress bar. - -@node nncp-log -@section nncp-log - -@example -$ nncp-log [options] -@end example - -Parse @ref{Log, log} file and print out its records in short -human-readable form. - -@node nncp-pkt -@section nncp-pkt - -@example -$ nncp-pkt [options] < pkt -$ nncp-pkt [options] [-decompress] -dump < pkt > payload -$ nncp-pkt -overheads -@end example - -Low level packet parser. Normally it should not be used, but can help in -debugging. - -By default it will print packet's type, for example: -@example -Packet type: encrypted -Niceness: 64 -Sender: 2WHBV3TPZHDOZGUJEH563ZEK7M33J4UESRFO4PDKWD5KZNPROABQ -@end example - -If you specify @option{-dump} option and provide an @ref{Encrypted, -encrypted} packet, then it will verify and decrypt it to @code{stdout}. -Encrypted packets contain @ref{Plain, plain} ones, that also can be fed -to @command{nncp-pkt}: - -@example -Packet type: plain -Payload type: transitional -Path: VHMTRWDOXPLK7BR55ICZ5N32ZJUMRKZEMFNGGCEAXV66GG43PEBQ - -Packet type: plain -Payload type: mail -Path: stargrave@@stargrave.org -@end example - -And with the @option{-dump} option it will give you the actual payload -(the whole file, mail message, and so on). @option{-decompress} option -tries to zstd-decompress the data from plain packet (useful for mail -packets). - -@option{-overheads} options print encrypted, plain and size header overheads. - -@node nncp-reass -@section nncp-reass - -@example -$ nncp-reass [options] [-dryrun] [-keep] [-dump] [-stdout] FILE.nncp.meta -$ nncp-reass [options] [-dryrun] [-keep] @{-all | -node NODE@} -@end example - -Reassemble @ref{Chunked, chunked file} after @ref{nncp-toss, tossing}. - -When called with @option{FILE} option, this command will reassemble only -it. When called with @option{-node} option, this command will try to -reassemble all @file{.nncp.meta} files found in @option{NODE}'s -@ref{CfgIncoming, incoming} directory. When called with @option{-all} -option, then cycle through all known nodes to do the same. - -Reassembling process does the following: - -@enumerate -@item Parses @ref{Chunked, @file{.nncp.meta}} file. -@item Checks existence and size of every @file{.nncp.chunkXXX}. -@item Verifies integrity of every chunk. -@item Concatenates all chunks, simultaneously removing them from filesystem. -@end enumerate - -That process reads the whole data twice. Be sure to have free disk -space for at least one chunk. Decrypted chunk files as a rule are saved -in pseudo-random order, so removing them during reassembly process will -likely lead to filesystem fragmentation. Reassembly process on -filesystems with deduplication capability should be rather lightweight. - -If @option{-dryrun} option is specified, then only existence and -integrity checking are performed. - -If @option{-keep} option is specified, then no -@file{.nncp.meta}/@file{.nncp.chunkXXX} files are deleted during -reassembly process. - -@option{-stdout} option outputs reassembled file to @code{stdout}, -instead of saving to temporary file with renaming after. This could be -useful for reassembling on separate filesystem to lower fragmentation -effect, and/or separate storage device for higher performance. - -@option{-dump} option prints meta-file contents in human-friendly form. -It is useful mainly for debugging purposes. For example: -@example -Original filename: testfile -File size: 3.8 MiB (3987795 bytes) -Chunk size: 1.0 MiB (1048576 bytes) -Number of chunks: 4 -Checksums: - 0: eac60d819edf40b8ecdacd0b9a5a8c62de2d15eef3c8ca719eafa0be9b894017 - 1: 013a07e659f2e353d0e4339c3375c96c7fffaa2fa00875635f440bbc4631052a - 2: f4f883975a663f2252328707a30e71b2678f933b2f3103db8475b03293e4316e - 3: 0e9e229501bf0ca42d4aa07393d19406d40b179f3922a3986ef12b41019b45a3 -@end example - - Do not forget about @ref{ChunkedZFS, possible} ZFS deduplication issues. - -@node nncp-rm -@section nncp-rm - -@example -$ nncp-rm [options] -tmp -$ nncp-rm [options] -lock -$ nncp-rm [options] -node NODE -part -$ nncp-rm [options] -node NODE -seen -$ nncp-rm [options] -node NODE -nock -$ nncp-rm [options] -node NODE [-rx] [-tx] -$ nncp-rm [options] -node NODE -pkt PKT -@end example - -This command is aimed to delete various files from your spool directory: - -@itemize - -@item If @option{-tmp} option is specified, then it will delete all -temporary files in @file{spool/tmp} directory. Files may stay in it when -commands like @ref{nncp-file} fail for some reason. - -@item If @option{-lock} option is specified, then all @file{.lock} files -will be deleted in your spool directory. - -@item If @option{-pkt} option is specified, then @file{PKT} packet (its -Base32 name) will be deleted. This is useful when you see some packet -failing to be processed. - -@item When either @option{-rx} or @option{-tx} options are specified -(maybe both of them), then delete all packets from that given queues. -@option{-part} option deletes @file{.part}ly downloaded files. -@option{-seen} option deletes @file{.seen} files. @option{-nock} option -deletes non-checksummed (non-verified) @file{.nock} files. - -@item @option{-dryrun} option just prints what will be deleted. - -@item You can also select files that only have modification date older -than specified @option{-older} time units (@code{10s} (10 seconds), -@code{5m} (5 minutes), @code{12h} (12 hours), @code{2d} (2 days)). - -@end itemize - -@node nncp-stat -@section nncp-stat - -@example -$ nncp-stat [options] [-pkt] [-node NODE] -@end example - -Print current @ref{Spool, spool} statistics about unsent and unprocessed -packets. For each node (unless @option{-node} specified) and each -niceness level there will be printed how many packets (with the total -size) are in inbound (Rx) and outbound (Tx) queues. @option{-pkt} option -show information about each packet. - -@node nncp-toss -@section nncp-toss - -@example -$ nncp-toss [options] - [-node NODE] - [-dryrun] - [-cycle INT] - [-seen] - [-nofile] - [-nofreq] - [-noexec] - [-notrns] -@end example - -Perform "tossing" operation on all inbound packets. This is the tool -that decrypts all packets and processes all payload packets in them: -copies files, sends mails, sends out file requests and relays transition -packets. It should be run after each online/offline exchange. - -@option{-dryrun} option does not perform any writing and sending, just -tells what it will do. - -@option{-cycle} option tells not to quit, but to repeat tossing every -@option{INT} seconds in an infinite loop. That can be useful when -running this command as a daemon. - -@option{-seen} option creates empty @file{XXX.seen} file after -successful tossing of @file{XXX} packet. @ref{nncp-xfer}, -@ref{nncp-bundle}, @ref{nncp-daemon} and @ref{nncp-call} commands skip -inbound packets that has been already seen, processed and tossed. This -is helpful to prevent duplicates. - -@option{-nofile}, @option{-nofreq}, @option{-noexec}, @option{-notrns} -options allow to disable any kind of packet types processing. - -@node nncp-xfer -@section nncp-xfer - -@example -$ nncp-xfer [options] [-node NODE] [-mkdir] [-keep] [-rx|-tx] DIR -@end example - -Search for directory in @file{DIR} containing inbound packets for us and -move them to local @ref{Spool, spool} directory. Also search for known -neighbours directories and move locally queued outbound packets to them. -This command is used for offline packets transmission. - -If @option{-mkdir} option is specified, then outbound neighbour(s) -directories will be created. This is useful for the first time usage, -when storage device does not have any directories tree. - -If @option{-keep} option is specified, then keep copied files, do not -remove them. - -@option{-rx} option tells only to move inbound packets addressed to us. -@option{-tx} option tells exactly the opposite: move only outbound packets. - -@ref{nncp-cfgmin} could be useful for creating stripped minimalistic -configuration file version without any private keys. - -@file{DIR} directory has the following structure: -@file{RECIPIENT/SENDER/PACKET}, where @file{RECIPIENT} is Base32 encoded -destination node, @file{SENDER} is Base32 encoded sender node. - -Also look for @ref{nncp-bundle}, especially if you deal with CD-ROM and -tape drives. diff --git a/doc/comparison.ru.texi b/doc/comparison.ru.texi index 280220c..933d157 100644 --- a/doc/comparison.ru.texi +++ b/doc/comparison.ru.texi @@ -14,6 +14,7 @@ @item Передача почты @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @item Передача новостей @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab Нет @item Передача файлов @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab Нет +@item Мультвещательная передача @tab Нет @tab @strong{Да} @tab @strong{Да} @tab Нет @item Разбиение файлов на части @tab Нет @tab @strong{Да} @tab @strong{Да} @tab Нет @item Удалённое исполнение команд @tab @strong{Да} @tab Нет @tab @strong{Да} @tab Нет @item Возобновляемое скачивание @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab Нет diff --git a/doc/comparison.texi b/doc/comparison.texi index 7bf4e53..d4e7ec6 100644 --- a/doc/comparison.texi +++ b/doc/comparison.texi @@ -13,6 +13,7 @@ FidoNet} Technology Networks) and @url{https://en.wikipedia.org/wiki/SMTP, SMTP} @item Mail transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @item News transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab No @item File transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab No +@item Multicast transmission @tab No @tab @strong{Yes} @tab @strong{Yes} @tab No @item Chunked files @tab No @tab @strong{Yes} @tab @strong{Yes} @tab No @item Remote command execution @tab @strong{Yes} @tab No @tab @strong{Yes} @tab No @item Resumable downloads @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab No diff --git a/doc/index.texi b/doc/index.texi index 27be3bb..000dab9 100644 --- a/doc/index.texi +++ b/doc/index.texi @@ -43,6 +43,7 @@ There are also articles about its usage outside this website: * Installation:: * Configuration:: * Call configuration: Call. +* Multicast areas: Multicast. * Integration:: * Commands:: * Administration:: @@ -67,17 +68,18 @@ There are also articles about its usage outside this website: @include news.texi @include russian.texi @include install.texi -@include cfg.texi +@include cfg/index.texi @include call.texi +@include multicast.texi @include integration.texi -@include cmds.texi +@include cmd/index.texi @include admin.texi @include niceness.texi @include chunked.texi @include bundles.texi @include spool.texi @include log.texi -@include pkt.texi +@include pkt/index.texi @include mth.texi @include sp.texi @include mcd.texi diff --git a/doc/multicast.texi b/doc/multicast.texi new file mode 100644 index 0000000..c330426 --- /dev/null +++ b/doc/multicast.texi @@ -0,0 +1,148 @@ +@node Multicast +@unnumbered Multicast areas + +NNCP has ability to multicast packets: send single packet to multiple +recipients, which also can send it further to others. It can also be +called echomail (like in FidoNet networks) or newsgroup (like in Usenet +networks). + +@anchor{Area} +Each multicast group is identified by so-called @strong{area}. Area +consists of private/public Curve25519 keypairs for @ref{Encrypted area, +packets encryption}, identity (BLAKE2b-256 hash of the public key) and +possible subscribers. + +You can make either file or exec transmissions to the areas. Those +ordinary file/exec packets are double wrapped in: + +@itemize +@item encrypted packet, securing the actual packet contents from +participants not having area's keypairs (but still being able to relay +that encrypted packet to the others) +@item area packet, containing area's identity, telling that tossing node +can should it to the subscribers further +@end itemize + +Area's message identity (@code{MsgHash}) is the hash of the encrypted +packet header. Because the area packet, containing the encrypted packet, +is relayed as-is without any modifications, that area message's hash +will be the same on each node it reaches. + +@ref{nncp-toss, Tosser}'s algorithm of processing the area packet is +following: + +@itemize +@item check is it known area's identity (@code{AREA}). + Fail/skip if it is unknown +@item hash encrypted packet's header, getting the @code{MsgHash} +@item for each area's subscribers: + @itemize + @item check if that message was already seen (sent or received from) + before by the destination node: check existence of + @file{SPOOL/NODE/area/AREA/MsgHash.seen} file. Skip that node if + it exists + @item if subscriber's node is not the one we received the packet + from, then create outgoing encrypted packet to it, with that + area packet inside + @item create corresponding @file{MsgHash.seen} file + @item "rewind" the outer encrypted file to the beginning and repeat + the whole cycle again, while all of subscribers will "seen" that + area's message. + + Expensive signature verification and shared key computation + procedures are skipped in the following cycles -- only symmetric + cryptography will be in use, having negligible CPU resource + consumption. + @end itemize +@item check if we have seen that area's message before by looking at + @file{SPOOL/SELF/area/AREA/MsgHash.seen}. If so, remove the packet, + because it is just a ordinary possible duplicate, finish its processing +@item check if we have got corresponding area's private key. If no key + exists, then remove the packet, finish its processing -- we just + relay it further without being able to read it +@item look if area's encrypted packet's sender is known to us. If + neither it is known, nor we have @code{allow-unknown} configuration + option set for that area, then fail +@item otherwise start decryption procedure, possibly ignoring the + sender's signature verification if it is unknown +@item fed the decrypted contents to the toss-procedure as an ordinary + plain packet, receiving files or exec calls +@item mark area's message as the seen one, remove the packet, finish + processing +@end itemize + +Because outgoing packets creation for each subscriber can be time and +(disk) resource consuming, we can suddenly fail. It would be bad if we +will loose the possibility to retry the multicasting process again. So +we have got to save somehow outgoing area's message in permanent +storage, while outgoing copies are created. That is why the initial (not +relaying) message to the area is sent to the @strong{self} and processed +by the @ref{nncp-toss, tosser} to create necessary outgoing message +copies. Because message to myself is also encrypted, area's message is +encrypted and secured and noone sees plaintext @code{MsgHash}, knowing +that you either originated or have that message on the disk. + +For example we have got 4 nodes participating in the single area and +let's send file to that area from the @code{nodeA}: + +@example +nodeA -> subs: ["nodeB", "nodeD"] +nodeB -> subs: ["nodeC", "nodeD", "nodeA"], no keys +nodeC -> subs: ["nodeB"] +nodeD -> subs: ["nodeA", "nodeB"] +@end example + +@example +A -- B -- C +\ / + \ / + D +@end example + +@example +$ nncp-file nodelist-20210704.rec area:nodelist-updates: +$ nncp-toss -node self +@end example + +@enumerate +@item +@command{nncp-file} creates an encrypted packet with area packet and +encrypted packet inside it, with our own @code{self} node as a recipient +(in the @file{SPOOL/SELF/tx} directory). It also creates the +@file{SPOOL/SELF/area/AREA/MSGHASH.seen} file. + +@item +@command{nncp-toss} sees @file{tx/} file and "opens" it, applying the +area message tossing procedure as described above. That will create +outgoing packets in @file{SPOOL/nodeB/tx} and @file{SPOOL/nodeD/tx} +directories with @file{SPOOL/nodeB/area/AREA/MSGHASH.seen} +@file{SPOOL/nodeD/area/AREA/MSGHASH.seen} files. Because we already have +@file{SPOOL/SELF/area/AREA/MSGHASH.seen}, that packet is removed then. + +@item +When @code{nodeB} receives the encrypted packet, it sees the area one +inside. It copies/relays it to the @code{nodeC} and @code{nodeD}. It can +not read area's message because it lacks the keys. + +@item +@code{nodeC} does not relay it to anyone. Just stores +@file{nodelist-20210704.rec} in the incoming directory. + +@item +@code{nodeD} receives packets from both @code{nodeA} and @code{nodeB}. +Only one of them processed, and other is ignored because corresponding +@file{MSGHASH.seen} file will exist. + +If @code{nodeD} will receive packet from the @code{nodeB} first, it will +relay it to the @code{nodeA} also, that will silently remove it when +tossing, because it was already seen. + +@strong{TODO}: we must not relay packet to the node also presenting as +the sender of the area's message. Obviously it has seen it. + +@item +When @code{nodeC} sends message to the area, then @code{nodeA} will +receive it twice from @code{nodeB} and @code{nodeD}, ignoring one of +them during tossing. + +@end enumerate diff --git a/doc/news.ru.texi b/doc/news.ru.texi index baa2310..3e0fe64 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -1,6 +1,28 @@ @node Новости @section Новости +@node Релиз 7.1.0 +@subsection Релиз 7.1.0 +@itemize + +@item +Исправлено некорректное генерирование @file{.hdr} при использовании +транзитных пакетов. + +@item +У @command{nncp-rm} команды появилась @option{-all} опция, применяемая +ко всем нодам сразу. + +@item +@command{nncp-pkt} может парсить @file{.hdr} файлы. + +@item +Появилась возможность мультивещательной (multicast) рассылки пакетов. +Реализовано всего лишь дополнительным типом простых пакетов и изменением +@command{nncp-toss}, @command{nncp-file} и @command{nncp-exec} команд. + +@end itemize + @node Релиз 7.0.0 @subsection Релиз 7.0.0 @itemize diff --git a/doc/news.texi b/doc/news.texi index 86e3754..cfa36c4 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -3,6 +3,27 @@ See also this page @ref{Новости, on russian}. +@node Release 7_1_0 +@section Release 7.1.0 +@itemize + +@item +Fixed invalid @file{.hdr} generation when transitional packets are used. + +@item +@option{-all} option appeared in @command{nncp-rm} command, applying to +all the nodes at once. + +@item +@command{nncp-pkt} can parse @file{.hdr} files. + +@item +Multicasting areas feature appeared. Implemented merely by an additional +plain packet type with @command{nncp-toss}, @command{nncp-file} and +@command{nncp-exec} commands modification. + +@end itemize + @node Release 7_0_0 @section Release 7.0.0 @itemize diff --git a/doc/pkt/area.texi b/doc/pkt/area.texi new file mode 100644 index 0000000..172c09c --- /dev/null +++ b/doc/pkt/area.texi @@ -0,0 +1,23 @@ +@node Encrypted area +@section Encrypted area packet + +@ref{Multicast} area messages contains the encrypted packet, that is +completely similar and have the same format as an ordinary +@ref{Encrypted, encrypted packet}. But instead of the node's identity, +area's identity is used as a recipient. + +For example when @code{nodeA} sends multicast packet with file +transmission and @code{nodeB} is the area's subscriber, then +@code{nodeA} has an encrypted packet to the @code{nodeB} in the outgoing +spool directory: + +@verbatim +ENCRYPTED PACKET (sender=nodeA, recipient=nodeB) WRAPS + PLAIN PACKET (type=area, path=AREA ID) WRAPS + ENCRYPTED PACKET (sender=nodeA, recipient=AREA) WRAPS <-- MsgHash + PLAIN PACKET (type=file, path=FILENAME) WRAPS + FILE CONTENTS +@end verbatim + +Area's message identity is the BLAKE2b-256 hash of header of the area's +packet encrypted packet. diff --git a/doc/pkt.texi b/doc/pkt/encrypted.texi similarity index 61% rename from doc/pkt.texi rename to doc/pkt/encrypted.texi index b479113..b036147 100644 --- a/doc/pkt.texi +++ b/doc/pkt/encrypted.texi @@ -1,76 +1,3 @@ -@node Packet -@unnumbered Packet format - -All packets are -@url{https://tools.ietf.org/html/rfc4506, XDR}-encoded structures. - -@menu -* Plain packet: Plain. -* Encrypted packet: Encrypted. -@end menu - -@node Plain -@section Plain packet - -Plain packet contains either the whole file, or file request (freq), or -transition packet or exec message. It is called "plain", because it -contains plaintext, but plain packets would never be stored on your hard -drive. - -@verbatim - HEADER -+--------------------------------------+--...---+ -| MAGIC | TYPE | NICE | PATHLEN | PATH | PAYLOAD| -+--------------------------------------+--...---+ -@end verbatim - -@multitable @columnfractions 0.2 0.3 0.5 -@headitem @tab XDR type @tab Value -@item Magic number @tab - 8-byte, fixed length opaque data @tab - @verb{|N N C P P 0x00 0x00 0x03|} -@item Payload type @tab - unsigned integer @tab - 0 (file), 1 (freq), 2 (exec), 3 (transition), 4 (exec-fat) -@item Niceness @tab - unsigned integer @tab - 1-255, preferred packet @ref{Niceness, niceness} level -@item Path length @tab - unsigned integer @tab - actual length of @emph{path} field's payload -@item Path @tab - 255 byte, fixed length opaque data @tab - @itemize - @item UTF-8 encoded destination path for file transfer - @item UTF-8 encoded source path for file request - @item UTF-8 encoded, zero byte separated, exec's arguments - @item Node's id the transition packet must be relayed on - @end itemize -@end multitable - -Path has fixed size because of hiding its actual length -- it is -valuable metadata. Payload is appended to the header -- it is not stored -as XDR field, because XDR has no ability to pass more than 4 GiB of -opaque data. Moreover most XDR libraries store fields in the memory in -practice. - -Depending on the packet's type, payload could store: - -@itemize -@item File contents -@item Destination path for freq -@item @url{https://facebook.github.io/zstd/, Zstandard} compressed exec body -@item Whole encrypted packet we need to relay on -@item Uncompressed exec body -@end itemize - -Also depending on packet's type, niceness level means: - -@itemize -@item Preferable niceness level for files sent by freq -@item @env{NNCP_NICE} variable's value passed during @ref{CfgExec} invocation. -@end itemize - @node Encrypted @section Encrypted packet diff --git a/doc/pkt/index.texi b/doc/pkt/index.texi new file mode 100644 index 0000000..7d29946 --- /dev/null +++ b/doc/pkt/index.texi @@ -0,0 +1,15 @@ +@node Packet +@unnumbered Packet format + +All packets are +@url{https://tools.ietf.org/html/rfc4506, XDR}-encoded structures. + +@menu +* Plain packet: Plain. +* Encrypted packet: Encrypted. +* Encrypted area packet: Encrypted area. +@end menu + +@include pkt/plain.texi +@include pkt/encrypted.texi +@include pkt/area.texi diff --git a/doc/pkt/plain.texi b/doc/pkt/plain.texi new file mode 100644 index 0000000..954b893 --- /dev/null +++ b/doc/pkt/plain.texi @@ -0,0 +1,144 @@ +@node Plain +@section Plain packet + +Plain packet contains either the whole file, or file request (freq), or +transition packet or exec message. It is called "plain", because it +contains plaintext, but plain packets would never be stored on your hard +drive. + +@verbatim + HEADER ++--------------------------------------+--...---+ +| MAGIC | TYPE | NICE | PATHLEN | PATH | PAYLOAD| ++--------------------------------------+--...---+ +@end verbatim + +@multitable @columnfractions 0.2 0.3 0.5 +@headitem @tab XDR type @tab Value +@item Magic number @tab + 8-byte, fixed length opaque data @tab + @verb{|N N C P P 0x00 0x00 0x03|} +@item Payload type @tab + unsigned integer @tab + @enumerate 0 + @item file (file transmission) + @item freq (file request) + @item exec (compressed exec) + @item trns (transition) + @item exec-fat (uncompressed exec) + @item area (@ref{Multicast, multicast} area message) + @end enumerate +@item Niceness @tab + unsigned integer @tab + 1-255, preferred packet @ref{Niceness, niceness} level +@item Path length @tab + unsigned integer @tab + actual length of @emph{path} field's payload +@item Path @tab + 255 byte, fixed length opaque data @tab + Depending on packet's type, path holds: + @itemize + @item UTF-8 encoded destination path for file transfer + @item UTF-8 encoded source path for file request + @item UTF-8 encoded, zero byte separated, exec's arguments + @item Node's id the transition packet must be relayed on + @item Multicast area's id + @end itemize +@end multitable + +Path has fixed size because of hiding its actual length -- it is +valuable metadata. Payload is appended to the header -- it is not stored +as XDR field, because XDR has no ability to pass more than 4 GiB of +opaque data. Moreover most XDR libraries store fields in the memory in +practice. + +Depending on the packet's type, payload could store: + +@itemize +@item File contents +@item Destination path for freq +@item Optionally @url{https://facebook.github.io/zstd/, Zstandard} + compressed exec body +@item Whole encrypted packet we need to relay on +@item Multicast area message wrap with another encrypted packet inside +@end itemize + +Also depending on packet's type, niceness level means: + +@itemize +@item Preferable niceness level for files sent by freq +@item @env{NNCP_NICE} variable's value passed during @ref{CfgExec} invocation. +@end itemize + +So plain packets can hold following paths and payloads: + +@table @code + +@item file +@example + +--------------- PATH ---------------+ +---- PAYLOAD ---+ + / \ / \ ++----------------------------------------+---------------...--+ +| FILENAME | 0x00 ... variable ... 0x00 | FILE CONTENTS | ++----------------------------------------+---------------...--+ + \ / + PATHLEN +@end example + +@item freq +@example + +--------------- PATH ---------------+ +---- PAYLOAD ---+ + / \ / \ ++----------------------------------------+---------------...--+ +| FILENAME | 0x00 ... variable ... 0x00 | FILENAME | ++----------------------------------------+---------------...--+ + \ / + PATHLEN +@end example + +@item exec +@example + +----------------------- PATH -------------------------+ +---- PAYLOAD ---+ + / \ / \ ++----------------------------------------------------------+---------------...--+ +| HANDLE | ARG0 0x00 ARG1 ...| 0x00 ... variable ... 0x00 | ZSTD DATA | ++----------------------------------------------------------+---------------...--+ + \ / + +-------- PATHLEN --------+ +@end example + +@item exec-fat +@example + +----------------------- PATH -------------------------+ +---- PAYLOAD ---+ + / \ / \ ++----------------------------------------------------------+---------------...--+ +| HANDLE | ARG0 0x00 ARG1 ...| 0x00 ... variable ... 0x00 | DATA | ++----------------------------------------------------------+---------------...--+ + \ / + +-------- PATHLEN --------+ +@end example + +@item trns +@example + +------- PATH ---------+ +---- PAYLOAD ---+ + / \ / \ ++--------------------------+---------------...--+ +| NODE ID | 0x00 ... 0x00 | ENCRYPTED PACKET | ++--------------------------+---------------...--+ + \ / + PATHLEN +@end example + +@item area +@example + +------- PATH ---------+ +---- PAYLOAD ---+ + / \ / \ ++--------------------------+---------------...--+ +| AREA ID | 0x00 ... 0x00 | ENCRYPTED PACKET | ++--------------------------+---------------...--+ + \ / + PATHLEN +@end example +See also @ref{Encrypted area, encrypted area packet}. + +@end table diff --git a/makedist.sh b/makedist.sh index c0b1c78..75b50e2 100755 --- a/makedist.sh +++ b/makedist.sh @@ -145,7 +145,7 @@ requests, Internet mail and commands transmission. All packets are integrity checked, end-to-end encrypted (E2EE), explicitly authenticated by known participants public keys. Onion encryption is applied to relayed packets. Each node acts both as a client and server, can use -push and poll behaviour model. +push and poll behaviour model. Also there is multicasting areas support. Out-of-box offline sneakernet/floppynet, dead drops, sequential and append-only CD-ROM/tape storages, air-gapped computers support. But @@ -191,7 +191,7 @@ NNCP (Node to Node copy) это набор утилит упрощающий б ключами участников. Луковичное (onion) шифрование применяется ко всем ретранслируемым пакетам. Каждый узел выступает одновременно в роли клиента и сервера, может использовать как push, так и poll модель -поведения. +поведения. А также есть поддержка мультивещательной рассылки пакетов. Поддержка из коробки offline флоппинета, тайников для сброса информации (dead drop), последовательных и только-для-записи CD-ROM/ленточных diff --git a/ports/nncp/Makefile b/ports/nncp/Makefile index 4440850..e2f6166 100644 --- a/ports/nncp/Makefile +++ b/ports/nncp/Makefile @@ -1,5 +1,5 @@ PORTNAME= nncp -DISTVERSION= 7.0.0 +DISTVERSION= 7.1.0 CATEGORIES= net MASTER_SITES= http://www.nncpgo.org/download/ diff --git a/ports/nncp/pkg-descr b/ports/nncp/pkg-descr index 1e1418f..9f921ed 100644 --- a/ports/nncp/pkg-descr +++ b/ports/nncp/pkg-descr @@ -8,7 +8,7 @@ requests, Internet mail and commands transmission. All packets are integrity checked, end-to-end encrypted (E2EE), explicitly authenticated by known participants public keys. Onion encryption is applied to relayed packets. Each node acts both as a client and server, can use -push and poll behaviour model. +push and poll behaviour model. Also there is multicasting areas support. Out-of-box offline sneakernet/floppynet, dead drops, sequential and append-only CD-ROM/tape storages, air-gapped computers support. But diff --git a/src/area.go b/src/area.go new file mode 100644 index 0000000..74c1324 --- /dev/null +++ b/src/area.go @@ -0,0 +1,69 @@ +/* +NNCP -- Node to Node copy, utilities for store-and-forward data exchange +Copyright (C) 2016-2021 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package nncp + +import ( + "errors" +) + +const AreaDir = "area" + +var ( + PktAreaOverhead int64 +) + +type AreaId [32]byte + +func (id AreaId) String() string { + return Base32Codec.EncodeToString(id[:]) +} + +type Area struct { + Name string + Id *AreaId + Pub *[32]byte + Prv *[32]byte + + Subs []*NodeId + + Exec map[string][]string + Incoming *string + + AllowUnknown bool +} + +func AreaIdFromString(raw string) (*AreaId, error) { + idRaw, err := Base32Codec.DecodeString(raw) + if err != nil { + return nil, err + } + if len(idRaw) != 32 { + return nil, errors.New("Invalid area id size") + } + areaId := new(AreaId) + copy(areaId[:], idRaw) + return areaId, nil +} + +func (ctx *Ctx) AreaName(id *AreaId) string { + area := ctx.AreaId2Area[*id] + if area == nil { + return id.String() + } + return area.Name +} diff --git a/src/call.go b/src/call.go index 5af4780..1e20aa2 100644 --- a/src/call.go +++ b/src/call.go @@ -45,6 +45,7 @@ type Call struct { AutoTossNoFreq bool AutoTossNoExec bool AutoTossNoTrns bool + AutoTossNoArea bool } func (ctx *Ctx) CallNode( diff --git a/src/cfg.go b/src/cfg.go index bb101ca..bacfd7e 100644 --- a/src/cfg.go +++ b/src/cfg.go @@ -91,6 +91,7 @@ type CallJSON struct { AutoTossNoFreq *bool `json:"autotoss-nofreq,omitempty"` AutoTossNoExec *bool `json:"autotoss-noexec,omitempty"` AutoTossNoTrns *bool `json:"autotoss-notrns,omitempty"` + AutoTossNoArea *bool `json:"autotoss-noarea,omitempty"` } type NodeOurJSON struct { @@ -114,6 +115,19 @@ type NotifyJSON struct { Exec map[string]*FromToJSON `json:"exec,omitempty"` } +type AreaJSON struct { + Id string `json:"id"` + Pub string `json:"pub"` + Prv *string `json:"prv,omitempty"` + + Subs []string `json:"subs"` + + Exec map[string][]string `json:"exec,omitempty"` + Incoming *string `json:"incoming,omitempty"` + + AllowUnknown *bool `json:"allow-unknown,omitempty"` +} + type CfgJSON struct { Spool string `json:"spool"` Log string `json:"log"` @@ -129,6 +143,8 @@ type CfgJSON struct { MCDRxIfis []string `json:"mcd-listen"` MCDTxIfis map[string]int `json:"mcd-send"` + + Areas map[string]AreaJSON `json:"areas"` } func NewNode(name string, cfg NodeJSON) (*Node, error) { @@ -314,6 +330,9 @@ func NewNode(name string, cfg NodeJSON) (*Node, error) { if callCfg.AutoTossNoTrns != nil { call.AutoTossNoTrns = *callCfg.AutoTossNoTrns } + if callCfg.AutoTossNoArea != nil { + call.AutoTossNoArea = *callCfg.AutoTossNoArea + } calls = append(calls, &call) } @@ -414,6 +433,52 @@ func NewNodeOur(cfg *NodeOurJSON) (*NodeOur, error) { return &node, nil } +func NewArea(ctx *Ctx, name string, cfg *AreaJSON) (*Area, error) { + areaId, err := AreaIdFromString(cfg.Id) + if err != nil { + return nil, err + } + subs := make([]*NodeId, 0, len(cfg.Subs)) + for _, s := range cfg.Subs { + node, err := ctx.FindNode(s) + if err != nil { + return nil, err + } + subs = append(subs, node.Id) + } + area := Area{ + Name: name, + Id: areaId, + Pub: new([32]byte), + Subs: subs, + Exec: cfg.Exec, + Incoming: cfg.Incoming, + } + pub, err := Base32Codec.DecodeString(cfg.Pub) + if err != nil { + return nil, err + } + if len(pub) != 32 { + return nil, errors.New("Invalid pub size") + } + copy(area.Pub[:], pub) + if cfg.Prv != nil { + prv, err := Base32Codec.DecodeString(*cfg.Prv) + if err != nil { + return nil, err + } + if len(prv) != 32 { + return nil, errors.New("Invalid prv size") + } + area.Prv = new([32]byte) + copy(area.Prv[:], prv) + } + if cfg.AllowUnknown != nil { + area.AllowUnknown = *cfg.AllowUnknown + } + return &area, nil +} + func CfgParse(data []byte) (*Ctx, error) { var err error if bytes.Compare(data[:8], MagicNNCPBv3.B[:]) == 0 { @@ -528,5 +593,15 @@ func CfgParse(data []byte) (*Ctx, error) { ) } } + ctx.AreaId2Area = make(map[AreaId]*Area, len(cfgJSON.Areas)) + ctx.AreaName2Id = make(map[string]*AreaId, len(cfgJSON.Areas)) + for name, areaJSON := range cfgJSON.Areas { + area, err := NewArea(&ctx, name, &areaJSON) + if err != nil { + return nil, err + } + ctx.AreaId2Area[*area.Id] = area + ctx.AreaName2Id[name] = area.Id + } return &ctx, nil } diff --git a/src/cmd/nncp-call/main.go b/src/cmd/nncp-call/main.go index 5d632d4..09c3e18 100644 --- a/src/cmd/nncp-call/main.go +++ b/src/cmd/nncp-call/main.go @@ -66,6 +66,7 @@ func main() { autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") + autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -174,6 +175,7 @@ func main() { *autoTossNoFreq, *autoTossNoExec, *autoTossNoTrns, + *autoTossNoArea, ) } diff --git a/src/cmd/nncp-caller/main.go b/src/cmd/nncp-caller/main.go index 4608484..d1a5b34 100644 --- a/src/cmd/nncp-caller/main.go +++ b/src/cmd/nncp-caller/main.go @@ -56,6 +56,7 @@ func main() { autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") + autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -200,6 +201,7 @@ func main() { call.AutoTossNoFreq || *autoTossNoFreq, call.AutoTossNoExec || *autoTossNoExec, call.AutoTossNoTrns || *autoTossNoTrns, + call.AutoTossNoArea || *autoTossNoArea, ) } diff --git a/src/cmd/nncp-cfgnew/main.go b/src/cmd/nncp-cfgnew/main.go index f992601..74f9ae6 100644 --- a/src/cmd/nncp-cfgnew/main.go +++ b/src/cmd/nncp-cfgnew/main.go @@ -19,11 +19,17 @@ along with this program. If not, see . package main import ( + "crypto/rand" + "encoding/json" "flag" "fmt" "log" "os" + "github.com/hjson/hjson-go" + "golang.org/x/crypto/blake2b" + "golang.org/x/crypto/nacl/box" + "go.cypherpunks.ru/nncp/v7" ) @@ -35,6 +41,7 @@ func usage() { func main() { var ( + areaName = flag.String("area", "", "Generate area's keypairs") noComments = flag.Bool("nocomments", false, "Do not include descriptive comments") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") @@ -50,9 +57,75 @@ func main() { fmt.Println(nncp.VersionGet()) return } + if *areaName != "" { + pub, prv, err := box.GenerateKey(rand.Reader) + if err != nil { + log.Fatalln(err) + } + areaId := nncp.AreaId(blake2b.Sum256(pub[:])) + var cfgRaw string + if *noComments { + cfgRaw = fmt.Sprintf(`areas: { + %s: { + id: %s + # KEEP AWAY keypair from the nodes you want only participate in multicast + pub: %s + prv: %s + } +}`, + *areaName, + areaId.String(), + nncp.Base32Codec.EncodeToString(pub[:]), + nncp.Base32Codec.EncodeToString(prv[:]), + ) + } else { + cfgRaw = fmt.Sprintf(`areas: { + %s: { + id: %s + + # KEEP AWAY keypair from the nodes you want only participate in multicast + pub: %s + prv: %s + + # List of subscribers you should multicast area messages to + # subs: ["alice"] + + # Allow incoming files (from the area) saving in that directory + # incoming: /home/areas/%s/incoming + + # Allow incoming area commands execution + # exec: {sendmail: ["%s"]} + + # Allow unknown sender's message tossing (relaying will be made anyway) + # allow-unknown: true + } +}`, + *areaName, + areaId.String(), + nncp.Base32Codec.EncodeToString(pub[:]), + nncp.Base32Codec.EncodeToString(prv[:]), + *areaName, + nncp.DefaultSendmailPath, + ) + } + var cfgGeneral map[string]interface{} + if err = hjson.Unmarshal([]byte(cfgRaw), &cfgGeneral); err != nil { + panic(err) + } + marshaled, err := json.Marshal(cfgGeneral) + if err != nil { + panic(err) + } + var areas map[string]nncp.AreaJSON + if err = json.Unmarshal(marshaled, &areas); err != nil { + panic(err) + } + fmt.Println(cfgRaw) + return + } nodeOur, err := nncp.NewNodeGenerate() if err != nil { - panic(err) + log.Fatalln(err) } var cfgRaw string if *noComments { diff --git a/src/cmd/nncp-daemon/main.go b/src/cmd/nncp-daemon/main.go index f79f3ad..b0628fe 100644 --- a/src/cmd/nncp-daemon/main.go +++ b/src/cmd/nncp-daemon/main.go @@ -157,6 +157,7 @@ func main() { autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") + autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -208,6 +209,7 @@ func main() { *autoTossNoFreq, *autoTossNoExec, *autoTossNoTrns, + *autoTossNoArea, ) } <-nodeIdC // call completion @@ -273,6 +275,7 @@ func main() { *autoTossNoFreq, *autoTossNoExec, *autoTossNoTrns, + *autoTossNoArea, ) } <-nodeIdC // call completion diff --git a/src/cmd/nncp-exec/main.go b/src/cmd/nncp-exec/main.go index c379ef0..11a9907 100644 --- a/src/cmd/nncp-exec/main.go +++ b/src/cmd/nncp-exec/main.go @@ -24,6 +24,7 @@ import ( "fmt" "log" "os" + "strings" "go.cypherpunks.ru/nncp/v7" ) @@ -31,7 +32,9 @@ import ( func usage() { fmt.Fprintf(os.Stderr, nncp.UsageHeader()) fmt.Fprintf(os.Stderr, "nncp-exec -- send execution command\n\n") - fmt.Fprintf(os.Stderr, "Usage: %s [options] NODE HANDLE [ARG0 ARG1 ...]\nOptions:\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [options] NODE HANDLE [ARG0 ARG1 ...]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] %s:AREA HANDLE [ARG0 ARG1 ...]\nOptions:\n", + os.Args[0], nncp.AreaDir) flag.PrintDefaults() } @@ -93,9 +96,19 @@ func main() { log.Fatalln("Config lacks private keys") } - node, err := ctx.FindNode(flag.Arg(0)) - if err != nil { - log.Fatalln("Invalid NODE specified:", err) + var areaId *nncp.AreaId + var node *nncp.Node + if strings.HasPrefix(flag.Arg(0), nncp.AreaDir+":") { + areaId = ctx.AreaName2Id[flag.Arg(0)[len(nncp.AreaDir)+1:]] + if areaId == nil { + log.Fatalln("Unknown area specified") + } + node = ctx.Neigh[*ctx.SelfId] + } else { + node, err = ctx.FindNode(flag.Arg(0)) + if err != nil { + log.Fatalln("Invalid NODE specified:", err) + } } nncp.ViaOverride(*viaOverride, ctx, node) @@ -111,6 +124,7 @@ func main() { int64(*minSize)*1024, *useTmp, *noCompress, + areaId, ); err != nil { log.Fatalln(err) } diff --git a/src/cmd/nncp-file/main.go b/src/cmd/nncp-file/main.go index 6fbbdc9..6870aec 100644 --- a/src/cmd/nncp-file/main.go +++ b/src/cmd/nncp-file/main.go @@ -31,7 +31,9 @@ import ( func usage() { fmt.Fprintf(os.Stderr, nncp.UsageHeader()) fmt.Fprintf(os.Stderr, "nncp-file -- send file\n\n") - fmt.Fprintf(os.Stderr, "Usage: %s [options] SRC NODE:[DST]\nOptions:\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [options] SRC NODE:[DST]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] SRC %s:AREA:[DST]\nOptions:\n", + os.Args[0], nncp.AreaDir) flag.PrintDefaults() fmt.Fprint(os.Stderr, ` If SRC equals to -, then read data from stdin to temporary file. @@ -93,14 +95,30 @@ func main() { log.Fatalln("Config lacks private keys") } - splitted := strings.SplitN(flag.Arg(1), ":", 2) - if len(splitted) != 2 { + splitted := strings.Split(flag.Arg(1), ":") + if len(splitted) < 2 { usage() os.Exit(1) } - node, err := ctx.FindNode(splitted[0]) - if err != nil { - log.Fatalln("Invalid NODE specified:", err) + var areaId *nncp.AreaId + var node *nncp.Node + if splitted[0] == nncp.AreaDir { + if len(splitted) < 3 { + usage() + os.Exit(1) + } + areaId = ctx.AreaName2Id[splitted[1]] + if areaId == nil { + log.Fatalln("Unknown area specified") + } + node = ctx.Neigh[*ctx.SelfId] + splitted = splitted[2:] + } else { + node, err = ctx.FindNode(splitted[0]) + if err != nil { + log.Fatalln("Invalid NODE specified:", err) + } + splitted = splitted[1:] } nncp.ViaOverride(*viaOverride, ctx, node) @@ -127,10 +145,11 @@ func main() { node, nice, flag.Arg(0), - splitted[1], + strings.Join(splitted, ":"), chunkSize, minSize, nncp.MaxFileSize, + areaId, ); err != nil { log.Fatalln(err) } diff --git a/src/cmd/nncp-pkt/main.go b/src/cmd/nncp-pkt/main.go index fbfc6da..3cd59ec 100644 --- a/src/cmd/nncp-pkt/main.go +++ b/src/cmd/nncp-pkt/main.go @@ -40,7 +40,7 @@ func usage() { fmt.Fprintln(os.Stderr, "Packet is read from stdin.") } -func doPlain(pkt nncp.Pkt, dump, decompress bool) { +func doPlain(ctx *nncp.Ctx, pkt nncp.Pkt, dump, decompress bool) { if dump { bufW := bufio.NewWriter(os.Stdout) var r io.Reader @@ -68,10 +68,12 @@ func doPlain(pkt nncp.Pkt, dump, decompress bool) { payloadType = "file request" case nncp.PktTypeExec: payloadType = "exec compressed" - case nncp.PktTypeExecFat: - payloadType = "exec uncompressed" case nncp.PktTypeTrns: payloadType = "transitional" + case nncp.PktTypeExecFat: + payloadType = "exec uncompressed" + case nncp.PktTypeArea: + payloadType = "area" } var path string switch pkt.Type { @@ -81,6 +83,15 @@ func doPlain(pkt nncp.Pkt, dump, decompress bool) { )) case nncp.PktTypeTrns: path = nncp.Base32Codec.EncodeToString(pkt.Path[:pkt.PathLen]) + node, err := ctx.FindNode(path) + if err != nil { + path = fmt.Sprintf("%s (%s)", path, node.Name) + } + case nncp.PktTypeArea: + path = nncp.Base32Codec.EncodeToString(pkt.Path[:pkt.PathLen]) + if areaId, err := nncp.AreaIdFromString(path); err == nil { + path = fmt.Sprintf("%s (%s)", path, ctx.AreaName(areaId)) + } default: path = string(pkt.Path[:pkt.PathLen]) } @@ -91,25 +102,37 @@ func doPlain(pkt nncp.Pkt, dump, decompress bool) { return } -func doEncrypted(pktEnc nncp.PktEnc, dump bool, cfgPath string, beginning []byte) { - ctx, err := nncp.CtxFromCmdline(cfgPath, "", "", false, false, false, false) - if err != nil { - log.Fatalln("Error during initialization:", err) +func doEncrypted( + ctx *nncp.Ctx, + pktEnc nncp.PktEnc, + dump bool, + beginning []byte, +) { + senderName := "unknown" + senderNode := ctx.Neigh[*pktEnc.Sender] + if senderNode != nil { + senderName = senderNode.Name } + + recipientName := "unknown" + var area *nncp.Area + recipientNode := ctx.Neigh[*pktEnc.Recipient] + if recipientNode == nil { + area = ctx.AreaId2Area[nncp.AreaId(*pktEnc.Recipient)] + recipientName = "area " + area.Name + } else { + recipientName = recipientNode.Name + } + if !dump { - senderS := "unknown" - recipientS := "unknown" - if n, ok := ctx.Neigh[*pktEnc.Sender]; ok { - senderS = n.Name - } - if n, ok := ctx.Neigh[*pktEnc.Recipient]; ok { - recipientS = n.Name - } - fmt.Printf( - "Packet type: encrypted\nNiceness: %s (%d)\nSender: %s (%s)\nRecipient: %s (%s)\n", + fmt.Printf(`Packet type: encrypted +Niceness: %s (%d) +Sender: %s (%s) +Recipient: %s (%s) +`, nncp.NicenessFmt(pktEnc.Nice), pktEnc.Nice, - pktEnc.Sender, senderS, - pktEnc.Recipient, recipientS, + pktEnc.Sender, senderName, + pktEnc.Recipient, recipientName, ) return } @@ -117,15 +140,24 @@ func doEncrypted(pktEnc nncp.PktEnc, dump bool, cfgPath string, beginning []byte log.Fatalln("Config lacks private keys") } bufW := bufio.NewWriter(os.Stdout) - if _, _, err = nncp.PktEncRead( - ctx.Self, - ctx.Neigh, - io.MultiReader( - bytes.NewReader(beginning), - bufio.NewReader(os.Stdin), - ), - bufW, - ); err != nil { + var err error + if area == nil { + _, _, _, err = nncp.PktEncRead( + ctx.Self, ctx.Neigh, + io.MultiReader(bytes.NewReader(beginning), bufio.NewReader(os.Stdin)), + bufW, senderNode != nil, nil, + ) + } else { + areaNode := nncp.NodeOur{Id: new(nncp.NodeId), ExchPrv: new([32]byte)} + copy(areaNode.Id[:], area.Id[:]) + copy(areaNode.ExchPrv[:], area.Prv[:]) + _, _, _, err = nncp.PktEncRead( + &areaNode, ctx.Neigh, + io.MultiReader(bytes.NewReader(beginning), bufio.NewReader(os.Stdin)), + bufW, senderNode != nil, nil, + ) + } + if err != nil { log.Fatalln(err) } if err = bufW.Flush(); err != nil { @@ -154,6 +186,11 @@ func main() { return } + ctx, err := nncp.CtxFromCmdline(*cfgPath, "", "", false, false, false, false) + if err != nil { + log.Fatalln("Error during initialization:", err) + } + if *overheads { fmt.Printf( "Plain: %d\nEncrypted: %d\nSize: %d\n", @@ -180,7 +217,7 @@ func main() { case nncp.MagicNNCPEv4.B: log.Fatalln(nncp.MagicNNCPEv4.TooOld()) case nncp.MagicNNCPEv5.B: - doEncrypted(pktEnc, *dump, *cfgPath, beginning[:nncp.PktEncOverhead]) + doEncrypted(ctx, pktEnc, *dump, beginning[:nncp.PktEncOverhead]) return } } @@ -196,7 +233,7 @@ func main() { case nncp.MagicNNCPPv2.B: log.Fatalln(nncp.MagicNNCPPv2.TooOld()) case nncp.MagicNNCPPv3.B: - doPlain(pkt, *dump, *decompress) + doPlain(ctx, pkt, *dump, *decompress) return } } diff --git a/src/cmd/nncp-rm/main.go b/src/cmd/nncp-rm/main.go index f8f4289..aa64228 100644 --- a/src/cmd/nncp-rm/main.go +++ b/src/cmd/nncp-rm/main.go @@ -37,12 +37,13 @@ func usage() { fmt.Fprintf(os.Stderr, "nncp-rm -- remove packet\n\n") fmt.Fprintf(os.Stderr, "Usage: %s [options] -tmp\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s [options] -lock\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -part\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -seen\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -nock\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -hdr\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE {-rx|-tx}\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -pkt PKT\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -part\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -seen\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -nock\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -hdr\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -area\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} {-rx|-tx}\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -pkt PKT\n", os.Args[0]) fmt.Fprintln(os.Stderr, "-older option's time units are: (s)econds, (m)inutes, (h)ours, (d)ays") fmt.Fprintln(os.Stderr, "Options:") flag.PrintDefaults() @@ -51,6 +52,7 @@ func usage() { func main() { var ( cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") + doAll = flag.Bool("all", false, "Apply remove rules to all nodes") doTmp = flag.Bool("tmp", false, "Remove all temporary files") doHdr = flag.Bool("hdr", false, "Remove all .hdr files") doLock = flag.Bool("lock", false, "Remove all lock files") @@ -60,6 +62,7 @@ func main() { doPart = flag.Bool("part", false, "Remove only .part files") doSeen = flag.Bool("seen", false, "Remove only .seen files") doNoCK = flag.Bool("nock", false, "Remove only .nock files") + doArea = flag.Bool("area", false, "Remove only area/*.seen files") older = flag.String("older", "", "XXX{smhd}: only older than XXX number of time units") dryRun = flag.Bool("dryrun", false, "Do not actually remove files") pktRaw = flag.String("pkt", "", "Packet to remove") @@ -165,70 +168,116 @@ func main() { } return } + var nodeId *nncp.NodeId if *nodeRaw == "" { - usage() - os.Exit(1) - } - node, err := ctx.FindNode(*nodeRaw) - if err != nil { - log.Fatalln("Invalid -node specified:", err) + if !*doAll { + usage() + os.Exit(1) + } + } else { + nodeId, err = nncp.NodeIdFromString(*nodeRaw) + if err != nil { + log.Fatalln("Invalid -node specified:", err) + } } - remove := func(xx nncp.TRxTx) error { - return filepath.Walk( - filepath.Join(ctx.Spool, node.Id.String(), string(xx)), - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - logMsg := func(les nncp.LEs) string { - return fmt.Sprintf("File %s: removed", path) - } - if now.Sub(info.ModTime()) < oldBoundary { - ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: path}}, func(les nncp.LEs) string { - return fmt.Sprintf("File %s: too fresh, skipping", path) - }) - return nil - } - if (*doSeen && strings.HasSuffix(info.Name(), nncp.SeenSuffix)) || - (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || - (*doHdr && strings.HasSuffix(info.Name(), nncp.HdrSuffix)) || - (*doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix)) { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) - if *dryRun { - return nil + for _, node := range ctx.Neigh { + if nodeId != nil && node.Id != nodeId { + continue + } + remove := func(xx nncp.TRxTx) error { + return filepath.Walk( + filepath.Join(ctx.Spool, node.Id.String(), string(xx)), + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err } - return os.Remove(path) - } - if *pktRaw != "" && filepath.Base(info.Name()) == *pktRaw { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) - if *dryRun { + if info.IsDir() { return nil } - return os.Remove(path) - } - if !*doSeen && !*doNoCK && !*doHdr && !*doPart && - (*doRx || *doTx) && - ((*doRx && xx == nncp.TRx) || (*doTx && xx == nncp.TTx)) { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) - if *dryRun { + logMsg := func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: path}}, func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", path) + }) return nil } - return os.Remove(path) - } - return nil - }) - } - if *pktRaw != "" || *doRx || *doSeen || *doNoCK || *doHdr || *doPart { - if err = remove(nncp.TRx); err != nil { - log.Fatalln("Can not remove:", err) + if (*doSeen && strings.HasSuffix(info.Name(), nncp.SeenSuffix)) || + (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || + (*doHdr && strings.HasSuffix(info.Name(), nncp.HdrSuffix)) || + (*doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix)) { + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + } + if *pktRaw != "" && filepath.Base(info.Name()) == *pktRaw { + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + } + if !*doSeen && !*doNoCK && !*doHdr && !*doPart && + (*doRx || *doTx) && + ((*doRx && xx == nncp.TRx) || (*doTx && xx == nncp.TTx)) { + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + } + return nil + }) } - } - if *pktRaw != "" || *doTx || *doHdr { - if err = remove(nncp.TTx); err != nil { - log.Fatalln("Can not remove:", err) + if *pktRaw != "" || *doRx || *doSeen || *doNoCK || *doHdr || *doPart { + if err = remove(nncp.TRx); err != nil { + log.Fatalln("Can not remove:", err) + } + } + if *pktRaw != "" || *doTx || *doHdr { + if err = remove(nncp.TTx); err != nil { + log.Fatalln("Can not remove:", err) + } + } + if *doArea { + if err = filepath.Walk( + filepath.Join(ctx.Spool, node.Id.String(), nncp.AreaDir), + func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if info.IsDir() { + return nil + } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: path}}, func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", path) + }) + return nil + } + if strings.HasSuffix(info.Name(), nncp.SeenSuffix) { + ctx.LogI( + "rm", + nncp.LEs{{K: "File", V: path}}, + func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + }, + ) + if *dryRun { + return nil + } + return os.Remove(path) + } + return nil + }); err != nil { + log.Fatalln("Can not remove:", err) + } } } } diff --git a/src/cmd/nncp-toss/main.go b/src/cmd/nncp-toss/main.go index d6b82fb..af3a59d 100644 --- a/src/cmd/nncp-toss/main.go +++ b/src/cmd/nncp-toss/main.go @@ -47,6 +47,7 @@ func main() { noFreq = flag.Bool("nofreq", false, "Do not process \"freq\" packets") noExec = flag.Bool("noexec", false, "Do not process \"exec\" packets") noTrns = flag.Bool("notrns", false, "Do not process \"transitional\" packets") + noArea = flag.Bool("noarea", false, "Do not process \"area\" packets") spoolPath = flag.String("spool", "", "Override path to spool") logPath = flag.String("log", "", "Override path to logfile") quiet = flag.Bool("quiet", false, "Print only errors") @@ -106,14 +107,18 @@ Cycle: } isBad = ctx.Toss( node.Id, + nncp.TRx, nice, - *dryRun, - *doSeen, - *noFile, - *noFreq, - *noExec, - *noTrns, - ) + *dryRun, *doSeen, *noFile, *noFreq, *noExec, *noTrns, *noArea, + ) || isBad + if nodeId == *ctx.SelfId { + isBad = ctx.Toss( + node.Id, + nncp.TTx, + nice, + *dryRun, false, true, true, true, true, *noArea, + ) || isBad + } } if *cycle > 0 { time.Sleep(time.Duration(*cycle) * time.Second) diff --git a/src/ctx.go b/src/ctx.go index fb26185..63d1fc2 100644 --- a/src/ctx.go +++ b/src/ctx.go @@ -36,6 +36,9 @@ type Ctx struct { Neigh map[NodeId]*Node Alias map[string]*NodeId + AreaId2Area map[AreaId]*Area + AreaName2Id map[string]*AreaId + Spool string LogPath string UmaskForce *int diff --git a/src/jobs.go b/src/jobs.go index 0819738..6e42877 100644 --- a/src/jobs.go +++ b/src/jobs.go @@ -20,6 +20,7 @@ package nncp import ( "bytes" "fmt" + "io" "os" "path/filepath" "strings" @@ -44,9 +45,9 @@ type Job struct { HshValue *[MTHSize]byte } -func (ctx *Ctx) HdrRead(fd *os.File) (*PktEnc, []byte, error) { +func (ctx *Ctx) HdrRead(r io.Reader) (*PktEnc, []byte, error) { var pktEnc PktEnc - _, err := xdr.Unmarshal(fd, &pktEnc) + _, err := xdr.Unmarshal(r, &pktEnc) if err != nil { return nil, nil, err } diff --git a/src/magic.go b/src/magic.go index 7814512..938d333 100644 --- a/src/magic.go +++ b/src/magic.go @@ -29,6 +29,10 @@ type Magic struct { } var ( + MagicNNCPAv1 = Magic{ + B: [8]byte{'N', 'N', 'C', 'P', 'A', 0, 0, 1}, + Name: "NNCPAv1 (area packet v1)", Till: "now", + } MagicNNCPBv1 = Magic{ B: [8]byte{'N', 'N', 'C', 'P', 'B', 0, 0, 1}, Name: "NNCPBv1 (EBlob v1)", Till: "1.0", diff --git a/src/nncp.go b/src/nncp.go index 1edc777..d3ad2a7 100644 --- a/src/nncp.go +++ b/src/nncp.go @@ -40,7 +40,7 @@ along with this program. If not, see .` const Base32Encoded32Len = 52 var ( - Version string = "7.0.0" + Version string = "7.1.0" Base32Codec *base32.Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) ) diff --git a/src/node.go b/src/node.go index b2a5985..aabf847 100644 --- a/src/node.go +++ b/src/node.go @@ -30,6 +30,8 @@ import ( "golang.org/x/crypto/nacl/box" ) +const DummyB32Id = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + type NodeId [blake2b.Size256]byte func (id NodeId) String() string { diff --git a/src/pkt.go b/src/pkt.go index 7d7e569..bd3fb23 100644 --- a/src/pkt.go +++ b/src/pkt.go @@ -44,6 +44,7 @@ const ( PktTypeExec PktType = iota PktTypeTrns PktType = iota PktTypeExecFat PktType = iota + PktTypeArea PktType = iota MaxPathSize = 1<<8 - 1 @@ -94,7 +95,7 @@ func init() { PktOverhead = int64(n) buf.Reset() - dummyId, err := NodeIdFromString("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + dummyId, err := NodeIdFromString(DummyB32Id) if err != nil { panic(err) } @@ -272,7 +273,7 @@ func PktEncWrite( return pktEncRaw, nil } -func TbsVerify(our *NodeOur, their *Node, pktEnc *PktEnc) ([]byte, bool, error) { +func TbsPrepare(our *NodeOur, their *Node, pktEnc *PktEnc) []byte { tbs := PktTbs{ Magic: MagicNNCPEv5.B, Nice: pktEnc.Nice, @@ -282,9 +283,14 @@ func TbsVerify(our *NodeOur, their *Node, pktEnc *PktEnc) ([]byte, bool, error) } var tbsBuf bytes.Buffer if _, err := xdr.Marshal(&tbsBuf, &tbs); err != nil { - return nil, false, err + panic(err) } - return tbsBuf.Bytes(), ed25519.Verify(their.SignPub, tbsBuf.Bytes(), pktEnc.Sign[:]), nil + return tbsBuf.Bytes() +} + +func TbsVerify(our *NodeOur, their *Node, pktEnc *PktEnc) ([]byte, bool, error) { + tbs := TbsPrepare(our, their, pktEnc) + return tbs, ed25519.Verify(their.SignPub, tbs, pktEnc.Sign[:]), nil } func PktEncRead( @@ -292,11 +298,13 @@ func PktEncRead( nodes map[NodeId]*Node, data io.Reader, out io.Writer, -) (*Node, int64, error) { + signatureVerify bool, + sharedKeyCached []byte, +) ([]byte, *Node, int64, error) { var pktEnc PktEnc _, err := xdr.Unmarshal(data, &pktEnc) if err != nil { - return nil, 0, err + return nil, nil, 0, err } switch pktEnc.Magic { case MagicNNCPEv1.B: @@ -312,51 +320,62 @@ func PktEncRead( err = BadMagic } if err != nil { - return nil, 0, err - } - their, known := nodes[*pktEnc.Sender] - if !known { - return nil, 0, errors.New("Unknown sender") + return nil, nil, 0, err } if *pktEnc.Recipient != *our.Id { - return nil, 0, errors.New("Invalid recipient") - } - tbsRaw, verified, err := TbsVerify(our, their, &pktEnc) - if err != nil { - return nil, 0, err - } - if !verified { - return their, 0, errors.New("Invalid signature") + return nil, nil, 0, errors.New("Invalid recipient") + } + var tbsRaw []byte + var their *Node + if signatureVerify { + their = nodes[*pktEnc.Sender] + if their == nil { + return nil, nil, 0, errors.New("Unknown sender") + } + var verified bool + tbsRaw, verified, err = TbsVerify(our, their, &pktEnc) + if err != nil { + return nil, nil, 0, err + } + if !verified { + return nil, their, 0, errors.New("Invalid signature") + } + } else { + tbsRaw = TbsPrepare(our, &Node{Id: pktEnc.Sender}, &pktEnc) } ad := blake3.Sum256(tbsRaw) sharedKey := new([32]byte) - curve25519.ScalarMult(sharedKey, our.ExchPrv, &pktEnc.ExchPub) + if sharedKeyCached == nil { + curve25519.ScalarMult(sharedKey, our.ExchPrv, &pktEnc.ExchPub) + } else { + copy(sharedKey[:], sharedKeyCached) + } key := make([]byte, chacha20poly1305.KeySize) blake3.DeriveKey(key, string(MagicNNCPEv5.B[:]), sharedKey[:]) aead, err := chacha20poly1305.New(key) if err != nil { - return their, 0, err + return sharedKey[:], their, 0, err } nonce := make([]byte, aead.NonceSize()) sizeBuf := make([]byte, 8+aead.Overhead()) if _, err = io.ReadFull(data, sizeBuf); err != nil { - return their, 0, err + return sharedKey[:], their, 0, err } sizeBuf, err = aead.Open(sizeBuf[:0], nonce, sizeBuf, ad[:]) if err != nil { - return their, 0, err + return sharedKey[:], their, 0, err } size := int64(binary.BigEndian.Uint64(sizeBuf)) lr := io.LimitedReader{R: data, N: size} written, err := aeadProcess(aead, nonce, ad[:], false, &lr, out) if err != nil { - return their, int64(written), err + return sharedKey[:], their, int64(written), err } if written != int(size) { - return their, int64(written), io.ErrUnexpectedEOF + return sharedKey[:], their, int64(written), io.ErrUnexpectedEOF } - return their, size, nil + return sharedKey[:], their, size, nil } diff --git a/src/pkt_test.go b/src/pkt_test.go index 079acbd..62efa71 100644 --- a/src/pkt_test.go +++ b/src/pkt_test.go @@ -112,7 +112,7 @@ func TestPktEncRead(t *testing.T) { var pt bytes.Buffer nodes := make(map[NodeId]*Node) nodes[*node1.Id] = node1.Their() - node, sizeGot, err := PktEncRead(node2, nodes, &ct, &pt) + _, node, sizeGot, err := PktEncRead(node2, nodes, &ct, &pt, true, nil) if err != nil { return false } diff --git a/src/toss.go b/src/toss.go index 7537d69..f23b583 100644 --- a/src/toss.go +++ b/src/toss.go @@ -38,6 +38,7 @@ import ( xdr "github.com/davecgh/go-xdr/xdr2" "github.com/dustin/go-humanize" "github.com/klauspost/compress/zstd" + "golang.org/x/crypto/blake2b" "golang.org/x/crypto/poly1305" ) @@ -64,609 +65,872 @@ func newNotification(fromTo *FromToJSON, subject string, body []byte) io.Reader return strings.NewReader(strings.Join(lines, "\n")) } -func (ctx *Ctx) Toss( - nodeId *NodeId, - nice uint8, - dryRun, doSeen, noFile, noFreq, noExec, noTrns bool, -) bool { - dirLock, err := ctx.LockDir(nodeId, "toss") - if err != nil { - return false +func pktSizeWithoutEnc(pktSize int64) int64 { + pktSize = pktSize - PktEncOverhead - PktOverhead - PktSizeOverhead + pktSizeBlocks := pktSize / (EncBlkSize + poly1305.TagSize) + if pktSize%(EncBlkSize+poly1305.TagSize) != 0 { + pktSize -= poly1305.TagSize } - defer ctx.UnlockDir(dirLock) - isBad := false + pktSize -= pktSizeBlocks * poly1305.TagSize + return pktSize +} + +var JobRepeatProcess = errors.New("needs processing repeat") + +func jobProcess( + ctx *Ctx, + pipeR *io.PipeReader, + pktName string, + les LEs, + sender *Node, + nice uint8, + pktSize uint64, + jobPath string, + decompressor *zstd.Decoder, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea bool, +) error { + defer pipeR.Close() sendmail := ctx.Neigh[*ctx.SelfId].Exec["sendmail"] - decompressor, err := zstd.NewReader(nil) + var pkt Pkt + _, err := xdr.Unmarshal(pipeR, &pkt) if err != nil { - panic(err) + ctx.LogE("rx-unmarshal", les, err, func(les LEs) string { + return fmt.Sprintf("Tossing %s/%s: unmarshal", sender.Name, pktName) + }) + return err } - defer decompressor.Close() - for job := range ctx.Jobs(nodeId, TRx) { - pktName := filepath.Base(job.Path) - les := LEs{ - {"Node", job.PktEnc.Sender}, - {"Pkt", pktName}, - {"Nice", int(job.PktEnc.Nice)}, - } - if job.PktEnc.Nice > nice { - ctx.LogD("rx-too-nice", les, func(les LEs) string { - return fmt.Sprintf( - "Tossing %s/%s: too nice: %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - NicenessFmt(job.PktEnc.Nice), - ) - }) - continue + les = append(les, LE{"Size", int64(pktSize)}) + ctx.LogD("rx", les, func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s (%s)", + sender.Name, pktName, + humanize.IBytes(pktSize), + ) + }) + switch pkt.Type { + case PktTypeExec, PktTypeExecFat: + if noExec { + return nil } - fd, err := os.Open(job.Path) - if err != nil { - ctx.LogE("rx-open", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing %s/%s: opening %s", - ctx.NodeName(job.PktEnc.Sender), pktName, job.Path, - ) - }) - isBad = true - continue + path := bytes.Split(pkt.Path[:int(pkt.PathLen)], []byte{0}) + handle := string(path[0]) + args := make([]string, 0, len(path)-1) + for _, p := range path[1:] { + args = append(args, string(p)) } - - pipeR, pipeW := io.Pipe() - go func(job Job) error { - pipeWB := bufio.NewWriter(pipeW) - _, _, err := PktEncRead(ctx.Self, ctx.Neigh, bufio.NewReader(fd), pipeWB) - fd.Close() // #nosec G104 - if err != nil { - return pipeW.CloseWithError(err) - } - if err = pipeWB.Flush(); err != nil { - return pipeW.CloseWithError(err) - } - return pipeW.Close() - }(job) - var pkt Pkt - var pktSize int64 - var pktSizeBlocks int64 - if _, err = xdr.Unmarshal(pipeR, &pkt); err != nil { - ctx.LogE("rx-unmarshal", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing %s/%s: unmarshal", - ctx.NodeName(job.PktEnc.Sender), pktName, - ) - }) - isBad = true - goto Closing + argsStr := strings.Join(append([]string{handle}, args...), " ") + les = append(les, LE{"Type", "exec"}, LE{"Dst", argsStr}) + cmdline, exists := sender.Exec[handle] + if !exists || len(cmdline) == 0 { + err = errors.New("No handle found") + ctx.LogE( + "rx-no-handle", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing exec %s/%s (%s): %s", + sender.Name, pktName, + humanize.IBytes(pktSize), argsStr, + ) + }, + ) + return err } - pktSize = job.Size - PktEncOverhead - PktOverhead - PktSizeOverhead - pktSizeBlocks = pktSize / (EncBlkSize + poly1305.TagSize) - if pktSize%(EncBlkSize+poly1305.TagSize) != 0 { - pktSize -= poly1305.TagSize + if pkt.Type == PktTypeExec { + if err = decompressor.Reset(pipeR); err != nil { + log.Fatalln(err) + } } - pktSize -= pktSizeBlocks * poly1305.TagSize - les = append(les, LE{"Size", pktSize}) - ctx.LogD("rx", les, func(les LEs) string { - return fmt.Sprintf( - "Tossing %s/%s (%s)", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), + if !dryRun { + cmd := exec.Command(cmdline[0], append(cmdline[1:], args...)...) + cmd.Env = append( + cmd.Env, + "NNCP_SELF="+ctx.Self.Id.String(), + "NNCP_SENDER="+sender.Id.String(), + "NNCP_NICE="+strconv.Itoa(int(pkt.Nice)), ) - }) - - switch pkt.Type { - case PktTypeExec, PktTypeExecFat: - if noExec { - goto Closing - } - path := bytes.Split(pkt.Path[:int(pkt.PathLen)], []byte{0}) - handle := string(path[0]) - args := make([]string, 0, len(path)-1) - for _, p := range path[1:] { - args = append(args, string(p)) - } - argsStr := strings.Join(append([]string{handle}, args...), " ") - les = append(les, LE{"Type", "exec"}, LE{"Dst", argsStr}) - sender := ctx.Neigh[*job.PktEnc.Sender] - cmdline, exists := sender.Exec[handle] - if !exists || len(cmdline) == 0 { - ctx.LogE( - "rx-no-handle", les, errors.New("No handle found"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing exec %s/%s (%s): %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), argsStr, - ) - }, - ) - isBad = true - goto Closing - } if pkt.Type == PktTypeExec { - if err = decompressor.Reset(pipeR); err != nil { - log.Fatalln(err) - } + cmd.Stdin = decompressor + } else { + cmd.Stdin = pipeR } - if !dryRun { - cmd := exec.Command(cmdline[0], append(cmdline[1:], args...)...) - cmd.Env = append( - cmd.Env, - "NNCP_SELF="+ctx.Self.Id.String(), - "NNCP_SENDER="+sender.Id.String(), - "NNCP_NICE="+strconv.Itoa(int(pkt.Nice)), - ) - if pkt.Type == PktTypeExec { - cmd.Stdin = decompressor - } else { - cmd.Stdin = pipeR - } - output, err := cmd.Output() - if err != nil { - ctx.LogE("rx-hande", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing exec %s/%s (%s): %s: handling", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), argsStr, - ) - }) - isBad = true - goto Closing + output, err := cmd.Output() + if err != nil { + ctx.LogE("rx-hande", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing exec %s/%s (%s): %s: handling", + sender.Name, pktName, + humanize.IBytes(uint64(pktSize)), argsStr, + ) + }) + return err + } + if len(sendmail) > 0 && ctx.NotifyExec != nil { + notify, exists := ctx.NotifyExec[sender.Name+"."+handle] + if !exists { + notify, exists = ctx.NotifyExec["*."+handle] } - if len(sendmail) > 0 && ctx.NotifyExec != nil { - notify, exists := ctx.NotifyExec[sender.Name+"."+handle] - if !exists { - notify, exists = ctx.NotifyExec["*."+handle] - } - if exists { - cmd := exec.Command( - sendmail[0], - append(sendmail[1:], notify.To)..., - ) - cmd.Stdin = newNotification(notify, fmt.Sprintf( - "Exec from %s: %s", sender.Name, argsStr, - ), output) - if err = cmd.Run(); err != nil { - ctx.LogE("rx-notify", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing exec %s/%s (%s): %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), argsStr, - ) - }) - } + if exists { + cmd := exec.Command( + sendmail[0], + append(sendmail[1:], notify.To)..., + ) + cmd.Stdin = newNotification(notify, fmt.Sprintf( + "Exec from %s: %s", sender.Name, argsStr, + ), output) + if err = cmd.Run(); err != nil { + ctx.LogE("rx-notify", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing exec %s/%s (%s): %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), argsStr, + ) + }) } } } - ctx.LogI("rx", les, func(les LEs) string { - return fmt.Sprintf( - "Got exec from %s to %s (%s)", - ctx.NodeName(job.PktEnc.Sender), argsStr, - humanize.IBytes(uint64(pktSize)), - ) - }) - if !dryRun { - if doSeen { - if fd, err := os.Create(job.Path + SeenSuffix); err == nil { - fd.Close() // #nosec G104 - } - } - if err = os.Remove(job.Path); err != nil { - ctx.LogE("rx-notify", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing exec %s/%s (%s): %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), argsStr, - ) - }) - isBad = true - } else if ctx.HdrUsage { - os.Remove(job.Path + HdrSuffix) + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf( + "Got exec from %s to %s (%s)", + sender.Name, argsStr, + humanize.IBytes(pktSize), + ) + }) + if !dryRun && jobPath != "" { + if doSeen { + if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + fd.Close() // #nosec G104 } } + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx-notify", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing exec %s/%s (%s): %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), argsStr, + ) + }) + return err + } else if ctx.HdrUsage { + os.Remove(jobPath + HdrSuffix) + } + } - case PktTypeFile: - if noFile { - goto Closing - } - dst := string(pkt.Path[:int(pkt.PathLen)]) - les = append(les, LE{"Type", "file"}, LE{"Dst", dst}) - if filepath.IsAbs(dst) { - ctx.LogE( - "rx-non-rel", les, errors.New("non-relative destination path"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }, + case PktTypeFile: + if noFile { + return nil + } + dst := string(pkt.Path[:int(pkt.PathLen)]) + les = append(les, LE{"Type", "file"}, LE{"Dst", dst}) + if filepath.IsAbs(dst) { + err = errors.New("non-relative destination path") + ctx.LogE( + "rx-non-rel", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }, + ) + return err + } + incoming := sender.Incoming + if incoming == nil { + err = errors.New("incoming is not allowed") + ctx.LogE( + "rx-no-incoming", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }, + ) + return err + } + dir := filepath.Join(*incoming, path.Dir(dst)) + if err = os.MkdirAll(dir, os.FileMode(0777)); err != nil { + ctx.LogE("rx-mkdir", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: mkdir", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) - isBad = true - goto Closing + }) + return err + } + if !dryRun { + tmp, err := TempFile(dir, "file") + if err != nil { + ctx.LogE("rx-mktemp", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: mktemp", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err } - incoming := ctx.Neigh[*job.PktEnc.Sender].Incoming - if incoming == nil { - ctx.LogE( - "rx-no-incoming", les, errors.New("incoming is not allowed"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }, + les = append(les, LE{"Tmp", tmp.Name()}) + ctx.LogD("rx-tmp-created", les, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: created: %s", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, tmp.Name(), ) - isBad = true - goto Closing + }) + bufW := bufio.NewWriter(tmp) + if _, err = CopyProgressed( + bufW, pipeR, "Rx file", + append(les, LE{"FullSize", int64(pktSize)}), + ctx.ShowPrgrs, + ); err != nil { + ctx.LogE("rx-copy", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: copying", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err } - dir := filepath.Join(*incoming, path.Dir(dst)) - if err = os.MkdirAll(dir, os.FileMode(0777)); err != nil { - ctx.LogE("rx-mkdir", les, err, func(les LEs) string { + if err = bufW.Flush(); err != nil { + tmp.Close() // #nosec G104 + ctx.LogE("rx-flush", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: mkdir", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: flushing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) }) - isBad = true - goto Closing + return err } - if !dryRun { - tmp, err := TempFile(dir, "file") - if err != nil { - ctx.LogE("rx-mktemp", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: mktemp", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - isBad = true - goto Closing - } - les = append(les, LE{"Tmp", tmp.Name()}) - ctx.LogD("rx-tmp-created", les, func(les LEs) string { + if err = tmp.Sync(); err != nil { + tmp.Close() // #nosec G104 + ctx.LogE("rx-sync", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: created: %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, tmp.Name(), + "Tossing file %s/%s (%s): %s: syncing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) }) - bufW := bufio.NewWriter(tmp) - if _, err = CopyProgressed( - bufW, pipeR, "Rx file", - append(les, LE{"FullSize", pktSize}), - ctx.ShowPrgrs, - ); err != nil { - ctx.LogE("rx-copy", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: copying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - isBad = true - goto Closing - } - if err = bufW.Flush(); err != nil { - tmp.Close() // #nosec G104 - ctx.LogE("rx-flush", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: flushing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - isBad = true - goto Closing - } - if err = tmp.Sync(); err != nil { - tmp.Close() // #nosec G104 - ctx.LogE("rx-sync", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: syncing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - isBad = true - goto Closing - } - if err = tmp.Close(); err != nil { - ctx.LogE("rx-close", les, err, func(les LEs) string { + return err + } + if err = tmp.Close(); err != nil { + ctx.LogE("rx-close", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: closing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err + } + dstPathOrig := filepath.Join(*incoming, dst) + dstPath := dstPathOrig + dstPathCtr := 0 + for { + if _, err = os.Stat(dstPath); err != nil { + if os.IsNotExist(err) { + break + } + ctx.LogE("rx-stat", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: closing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: stating: %s", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, dstPath, ) }) - isBad = true - goto Closing + return err } - dstPathOrig := filepath.Join(*incoming, dst) - dstPath := dstPathOrig - dstPathCtr := 0 - for { - if _, err = os.Stat(dstPath); err != nil { - if os.IsNotExist(err) { - break - } - ctx.LogE("rx-stat", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: stating: %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, dstPath, - ) - }) - isBad = true - goto Closing + dstPath = dstPathOrig + "." + strconv.Itoa(dstPathCtr) + dstPathCtr++ + } + if err = os.Rename(tmp.Name(), dstPath); err != nil { + ctx.LogE("rx-rename", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: renaming", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err + } + if err = DirSync(*incoming); err != nil { + ctx.LogE("rx-dirsync", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: dirsyncing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err + } + les = les[:len(les)-1] // delete Tmp + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf( + "Got file %s (%s) from %s", + dst, humanize.IBytes(pktSize), sender.Name, + ) + }) + if !dryRun { + if jobPath != "" { + if doSeen { + if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + fd.Close() // #nosec G104 } - dstPath = dstPathOrig + "." + strconv.Itoa(dstPathCtr) - dstPathCtr++ } - if err = os.Rename(tmp.Name(), dstPath); err != nil { - ctx.LogE("rx-rename", les, err, func(les LEs) string { + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx-remove", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: renaming", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) }) - isBad = true + return err + } else if ctx.HdrUsage { + os.Remove(jobPath + HdrSuffix) } - if err = DirSync(*incoming); err != nil { - ctx.LogE("rx-dirsync", les, err, func(les LEs) string { + } + if len(sendmail) > 0 && ctx.NotifyFile != nil { + cmd := exec.Command( + sendmail[0], + append(sendmail[1:], ctx.NotifyFile.To)..., + ) + cmd.Stdin = newNotification(ctx.NotifyFile, fmt.Sprintf( + "File from %s: %s (%s)", + sender.Name, dst, humanize.IBytes(pktSize), + ), nil) + if err = cmd.Run(); err != nil { + ctx.LogE("rx-notify", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: dirsyncing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) }) - isBad = true } - les = les[:len(les)-1] // delete Tmp } - ctx.LogI("rx", les, func(les LEs) string { + } + + case PktTypeFreq: + if noFreq { + return nil + } + src := string(pkt.Path[:int(pkt.PathLen)]) + les := append(les, LE{"Type", "freq"}, LE{"Src", src}) + if filepath.IsAbs(src) { + err = errors.New("non-relative source path") + ctx.LogE( + "rx-non-rel", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing freq %s/%s (%s): %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), src, + ) + }, + ) + return err + } + dstRaw, err := ioutil.ReadAll(pipeR) + if err != nil { + ctx.LogE("rx-read", les, err, func(les LEs) string { return fmt.Sprintf( - "Got file %s (%s) from %s", - dst, humanize.IBytes(uint64(pktSize)), - ctx.NodeName(job.PktEnc.Sender), + "Tossing freq %s/%s (%s): %s: reading", + sender.Name, pktName, + humanize.IBytes(pktSize), src, ) }) - if !dryRun { + return err + } + dst := string(dstRaw) + les = append(les, LE{"Dst", dst}) + freqPath := sender.FreqPath + if freqPath == nil { + err = errors.New("freqing is not allowed") + ctx.LogE( + "rx-no-freq", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing freq %s/%s (%s): %s -> %s", + sender.Name, pktName, + humanize.IBytes(pktSize), src, dst, + ) + }, + ) + return err + } + if !dryRun { + err = ctx.TxFile( + sender, + pkt.Nice, + filepath.Join(*freqPath, src), + dst, + sender.FreqChunked, + sender.FreqMinSize, + sender.FreqMaxSize, + nil, + ) + if err != nil { + ctx.LogE("rx-tx", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing freq %s/%s (%s): %s -> %s: txing", + sender.Name, pktName, + humanize.IBytes(pktSize), src, dst, + ) + }) + return err + } + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf("Got file request %s to %s", src, sender.Name) + }) + if !dryRun { + if jobPath != "" { if doSeen { - if fd, err := os.Create(job.Path + SeenSuffix); err == nil { + if fd, err := os.Create(jobPath + SeenSuffix); err == nil { fd.Close() // #nosec G104 } } - if err = os.Remove(job.Path); err != nil { + if err = os.Remove(jobPath); err != nil { ctx.LogE("rx-remove", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: removing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing freq %s/%s (%s): %s -> %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), src, dst, ) }) - isBad = true + return err } else if ctx.HdrUsage { - os.Remove(job.Path + HdrSuffix) - } - if len(sendmail) > 0 && ctx.NotifyFile != nil { - cmd := exec.Command( - sendmail[0], - append(sendmail[1:], ctx.NotifyFile.To)..., - ) - cmd.Stdin = newNotification(ctx.NotifyFile, fmt.Sprintf( - "File from %s: %s (%s)", - ctx.Neigh[*job.PktEnc.Sender].Name, - dst, - humanize.IBytes(uint64(pktSize)), - ), nil) - if err = cmd.Run(); err != nil { - ctx.LogE("rx-notify", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - } + os.Remove(jobPath + HdrSuffix) } } - - case PktTypeFreq: - if noFreq { - goto Closing - } - src := string(pkt.Path[:int(pkt.PathLen)]) - les := append(les, LE{"Type", "freq"}, LE{"Src", src}) - if filepath.IsAbs(src) { - ctx.LogE( - "rx-non-rel", les, errors.New("non-relative source path"), - func(les LEs) string { + if len(sendmail) > 0 && ctx.NotifyFreq != nil { + cmd := exec.Command( + sendmail[0], + append(sendmail[1:], ctx.NotifyFreq.To)..., + ) + cmd.Stdin = newNotification(ctx.NotifyFreq, fmt.Sprintf( + "Freq from %s: %s", sender.Name, src, + ), nil) + if err = cmd.Run(); err != nil { + ctx.LogE("rx-notify", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, + "Tossing freq %s/%s (%s): %s -> %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), src, dst, ) - }, - ) - isBad = true - goto Closing + }) + } } - dstRaw, err := ioutil.ReadAll(pipeR) - if err != nil { - ctx.LogE("rx-read", les, err, func(les LEs) string { + } + + case PktTypeTrns: + if noTrns { + return nil + } + dst := new([MTHSize]byte) + copy(dst[:], pkt.Path[:int(pkt.PathLen)]) + nodeId := NodeId(*dst) + les := append(les, LE{"Type", "trns"}, LE{"Dst", nodeId}) + logMsg := func(les LEs) string { + return fmt.Sprintf( + "Tossing trns %s/%s (%s): %s", + sender.Name, pktName, + humanize.IBytes(pktSize), + nodeId.String(), + ) + } + node := ctx.Neigh[nodeId] + if node == nil { + err = errors.New("unknown node") + ctx.LogE("rx-unknown", les, err, logMsg) + return err + } + ctx.LogD("rx-tx", les, logMsg) + if !dryRun { + if err = ctx.TxTrns(node, nice, int64(pktSize), pipeR); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { + return logMsg(les) + ": txing" + }) + return err + } + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf( + "Got transitional packet from %s to %s (%s)", + sender.Name, + ctx.NodeName(&nodeId), + humanize.IBytes(pktSize), + ) + }) + if !dryRun && jobPath != "" { + if doSeen { + if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + fd.Close() // #nosec G104 + } + } + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s: reading", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, + "Tossing trns %s/%s (%s): %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), + ctx.NodeName(&nodeId), ) }) - isBad = true - goto Closing - } - dst := string(dstRaw) - les = append(les, LE{"Dst", dst}) - sender := ctx.Neigh[*job.PktEnc.Sender] - freqPath := sender.FreqPath - if freqPath == nil { - ctx.LogE( - "rx-no-freq", les, errors.New("freqing is not allowed"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s -> %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, dst, - ) - }, - ) - isBad = true - goto Closing - } - if !dryRun { - err = ctx.TxFile( - sender, - pkt.Nice, - filepath.Join(*freqPath, src), - dst, - sender.FreqChunked, - sender.FreqMinSize, - sender.FreqMaxSize, + return err + } else if ctx.HdrUsage { + os.Remove(jobPath + HdrSuffix) + } + } + + case PktTypeArea: + if noArea { + return nil + } + areaId := new(AreaId) + copy(areaId[:], pkt.Path[:int(pkt.PathLen)]) + les := append(les, LE{"Type", "area"}, LE{"Area", areaId}) + logMsg := func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s (%s): area %s", + sender.Name, pktName, + humanize.IBytes(pktSize), + ctx.AreaName(areaId), + ) + } + area := ctx.AreaId2Area[*areaId] + if area == nil { + err = errors.New("unknown area") + ctx.LogE("rx-area-unknown", les, err, logMsg) + return err + } + pktEnc, pktEncRaw, err := ctx.HdrRead(pipeR) + fullPipeR := io.MultiReader(bytes.NewReader(pktEncRaw), pipeR) + if err != nil { + ctx.LogE("rx-area-pkt-enc-read", les, err, logMsg) + return err + } + msgHashRaw := blake2b.Sum256(pktEncRaw) + msgHash := Base32Codec.EncodeToString(msgHashRaw[:]) + les = append(les, LE{"AreaMsg", msgHash}) + ctx.LogD("rx-area", les, logMsg) + + if dryRun { + for _, nodeId := range area.Subs { + node := ctx.Neigh[*nodeId] + lesEcho := append(les, LE{"Echo", nodeId}) + seenDir := filepath.Join( + ctx.Spool, nodeId.String(), AreaDir, area.Id.String(), ) - if err != nil { - ctx.LogE("rx-tx", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s -> %s: txing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, dst, - ) + seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + logMsgNode := func(les LEs) string { + return fmt.Sprintf( + "%s: echoing to: %s", logMsg(les), node.Name, + ) + } + if _, err := os.Stat(seenPath); err == nil { + ctx.LogD("rx-area-echo-seen", lesEcho, func(les LEs) string { + return logMsgNode(les) + ": already sent" }) - isBad = true - goto Closing + continue } + ctx.LogI("rx-area-echo", lesEcho, logMsgNode) } - ctx.LogI("rx", les, func(les LEs) string { - return fmt.Sprintf( - "Got file request %s to %s", - src, ctx.NodeName(job.PktEnc.Sender), + } else { + for _, nodeId := range area.Subs { + node := ctx.Neigh[*nodeId] + lesEcho := append(les, LE{"Echo", nodeId}) + seenDir := filepath.Join( + ctx.Spool, nodeId.String(), AreaDir, area.Id.String(), ) - }) - if !dryRun { - if doSeen { - if fd, err := os.Create(job.Path + SeenSuffix); err == nil { - fd.Close() // #nosec G104 + seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + logMsgNode := func(les LEs) string { + return fmt.Sprintf("%s: echo to: %s", logMsg(les), node.Name) + } + if _, err := os.Stat(seenPath); err == nil { + ctx.LogD("rx-area-echo-seen", lesEcho, func(les LEs) string { + return logMsgNode(les) + ": already sent" + }) + continue + } + if nodeId != sender.Id { + ctx.LogI("rx-area-echo", lesEcho, logMsgNode) + if _, err = ctx.Tx( + node, &pkt, nice, int64(pktSize), 0, fullPipeR, pktName, nil, + ); err != nil { + ctx.LogE("rx-area", lesEcho, err, logMsgNode) + return err } } - if err = os.Remove(job.Path); err != nil { - ctx.LogE("rx-remove", les, err, func(les LEs) string { + if err = os.MkdirAll(seenDir, os.FileMode(0777)); err != nil { + ctx.LogE("rx-area-mkdir", lesEcho, err, logMsgNode) + return err + } + if fd, err := os.Create(seenPath); err == nil { + fd.Close() + if err = DirSync(seenDir); err != nil { + ctx.LogE("rx-area-dirsync", les, err, logMsgNode) + return err + } + } else { + ctx.LogE("rx-area-touch", lesEcho, err, logMsgNode) + return err + } + return JobRepeatProcess + } + } + + seenDir := filepath.Join( + ctx.Spool, ctx.SelfId.String(), AreaDir, area.Id.String(), + ) + seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + if _, err := os.Stat(seenPath); err == nil { + ctx.LogD("rx-area-seen", les, func(les LEs) string { + return logMsg(les) + ": already seen" + }) + if !dryRun && jobPath != "" { + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx-area-remove", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s -> %s: removing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, dst, + "Tossing area %s/%s (%s): %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), + msgHash, ) }) - isBad = true + return err } else if ctx.HdrUsage { - os.Remove(job.Path + HdrSuffix) + os.Remove(jobPath + HdrSuffix) } - if len(sendmail) > 0 && ctx.NotifyFreq != nil { - cmd := exec.Command( - sendmail[0], - append(sendmail[1:], ctx.NotifyFreq.To)..., + } + return nil + } + + if area.Prv == nil { + ctx.LogD("rx-area-no-prv", les, func(les LEs) string { + return logMsg(les) + ": no private key for decoding" + }) + } else { + signatureVerify := true + if _, senderKnown := ctx.Neigh[*pktEnc.Sender]; !senderKnown { + if !area.AllowUnknown { + err = errors.New("unknown sender") + ctx.LogE( + "rx-area-unknown", + append(les, LE{"Sender", pktEnc.Sender}), + err, + func(les LEs) string { + return logMsg(les) + ": sender: " + pktEnc.Sender.String() + }, ) - cmd.Stdin = newNotification(ctx.NotifyFreq, fmt.Sprintf( - "Freq from %s: %s", sender.Name, src, - ), nil) - if err = cmd.Run(); err != nil { - ctx.LogE("rx-notify", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s -> %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, dst, - ) - }) - } + return err } + signatureVerify = false + } + areaNodeOur := NodeOur{Id: new(NodeId), ExchPrv: new([32]byte)} + copy(areaNodeOur.Id[:], area.Id[:]) + copy(areaNodeOur.ExchPrv[:], area.Prv[:]) + areaNode := Node{ + Id: new(NodeId), + Name: area.Name, + Incoming: area.Incoming, + Exec: area.Exec, } + copy(areaNode.Id[:], area.Id[:]) + pktName := fmt.Sprintf( + "area/%s/%s", + Base32Codec.EncodeToString(areaId[:]), msgHash, + ) - case PktTypeTrns: - if noTrns { - goto Closing - } - dst := new([MTHSize]byte) - copy(dst[:], pkt.Path[:int(pkt.PathLen)]) - nodeId := NodeId(*dst) - node, known := ctx.Neigh[nodeId] - les := append(les, LE{"Type", "trns"}, LE{"Dst", nodeId}) - logMsg := func(les LEs) string { - return fmt.Sprintf( - "Tossing trns %s/%s (%s): %s", - ctx.NodeName(job.PktEnc.Sender), + pipeR, pipeW := io.Pipe() + errs := make(chan error, 1) + go func() { + errs <- jobProcess( + ctx, + pipeR, pktName, - humanize.IBytes(uint64(pktSize)), - nodeId.String(), + les, + &areaNode, + nice, + uint64(pktSizeWithoutEnc(int64(pktSize))), + "", + decompressor, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, ) + }() + _, _, _, err = PktEncRead( + &areaNodeOur, + ctx.Neigh, + fullPipeR, + pipeW, + signatureVerify, + nil, + ) + if err != nil { + pipeW.CloseWithError(err) + go func() { <-errs }() + return err } - if !known { - ctx.LogE("rx-unknown", les, errors.New("unknown node"), logMsg) - isBad = true - goto Closing + pipeW.Close() + if err = <-errs; err != nil { + return err } - ctx.LogD("rx-tx", les, logMsg) - if !dryRun { - if err = ctx.TxTrns(node, job.PktEnc.Nice, pktSize, pipeR); err != nil { - ctx.LogE("rx", les, err, func(les LEs) string { - return logMsg(les) + ": txing" - }) - isBad = true - goto Closing + } + + if !dryRun && jobPath != "" { + if err = os.MkdirAll(seenDir, os.FileMode(0777)); err != nil { + ctx.LogE("rx-area-mkdir", les, err, logMsg) + return err + } + if fd, err := os.Create(seenPath); err == nil { + fd.Close() + if err = DirSync(seenDir); err != nil { + ctx.LogE("rx-area-dirsync", les, err, logMsg) + return err } } - ctx.LogI("rx", les, func(les LEs) string { + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing area %s/%s (%s): %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), + msgHash, + ) + }) + return err + } else if ctx.HdrUsage { + os.Remove(jobPath + HdrSuffix) + } + } + + default: + err = errors.New("unknown type") + ctx.LogE( + "rx-type-unknown", les, err, + func(les LEs) string { return fmt.Sprintf( - "Got transitional packet from %s to %s (%s)", - ctx.NodeName(job.PktEnc.Sender), - ctx.NodeName(&nodeId), - humanize.IBytes(uint64(pktSize)), + "Tossing %s/%s (%s)", + sender.Name, pktName, humanize.IBytes(pktSize), + ) + }, + ) + return err + } + return nil +} + +func (ctx *Ctx) Toss( + nodeId *NodeId, + xx TRxTx, + nice uint8, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea bool, +) bool { + dirLock, err := ctx.LockDir(nodeId, "toss") + if err != nil { + return false + } + defer ctx.UnlockDir(dirLock) + isBad := false + decompressor, err := zstd.NewReader(nil) + if err != nil { + panic(err) + } + defer decompressor.Close() + for job := range ctx.Jobs(nodeId, xx) { + pktName := filepath.Base(job.Path) + les := LEs{ + {"Node", job.PktEnc.Sender}, + {"Pkt", pktName}, + {"Nice", int(job.PktEnc.Nice)}, + } + if job.PktEnc.Nice > nice { + ctx.LogD("rx-too-nice", les, func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s: too nice: %s", + ctx.NodeName(job.PktEnc.Sender), pktName, + NicenessFmt(job.PktEnc.Nice), ) }) - if !dryRun { - if doSeen { - if fd, err := os.Create(job.Path + SeenSuffix); err == nil { - fd.Close() // #nosec G104 - } - } - if err = os.Remove(job.Path); err != nil { - ctx.LogE("rx", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing trns %s/%s (%s): %s: removing", - ctx.NodeName(job.PktEnc.Sender), - pktName, - humanize.IBytes(uint64(pktSize)), - ctx.NodeName(&nodeId), - ) - }) - isBad = true - } else if ctx.HdrUsage { - os.Remove(job.Path + HdrSuffix) - } - } + continue + } + fd, err := os.Open(job.Path) + if err != nil { + ctx.LogE("rx-open", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s: opening %s", + ctx.NodeName(job.PktEnc.Sender), pktName, job.Path, + ) + }) + isBad = true + continue + } + errs := make(chan error, 1) + var sharedKey []byte + Retry: + pipeR, pipeW := io.Pipe() + go func() { + errs <- jobProcess( + ctx, + pipeR, + pktName, + les, + ctx.Neigh[*job.PktEnc.Sender], + job.PktEnc.Nice, + uint64(pktSizeWithoutEnc(job.Size)), + job.Path, + decompressor, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, + ) + }() + pipeWB := bufio.NewWriter(pipeW) + sharedKey, _, _, err = PktEncRead( + ctx.Self, + ctx.Neigh, + bufio.NewReader(fd), + pipeWB, + sharedKey == nil, + sharedKey, + ) + if err != nil { + pipeW.CloseWithError(err) + } + if err := pipeWB.Flush(); err != nil { + pipeW.CloseWithError(err) + } + pipeW.Close() - default: - ctx.LogE( - "rx-type-unknown", les, errors.New("unknown type"), - func(les LEs) string { + if err != nil { + isBad = true + fd.Close() + go func() { <-errs }() + continue + } + if err = <-errs; err == JobRepeatProcess { + if _, err = fd.Seek(0, io.SeekStart); err != nil { + ctx.LogE("rx-seek", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing %s/%s (%s)", + "Tossing %s/%s: can not seek", ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), ) - }, - ) + }) + isBad = true + break + } + goto Retry + } else if err != nil { isBad = true } - Closing: - pipeR.Close() // #nosec G104 + fd.Close() } return isBad } @@ -674,7 +938,7 @@ func (ctx *Ctx) Toss( func (ctx *Ctx) AutoToss( nodeId *NodeId, nice uint8, - doSeen, noFile, noFreq, noExec, noTrns bool, + doSeen, noFile, noFreq, noExec, noTrns, noArea bool, ) (chan struct{}, chan bool) { finish := make(chan struct{}) badCode := make(chan bool) @@ -688,7 +952,9 @@ func (ctx *Ctx) AutoToss( default: } time.Sleep(time.Second) - bad = !ctx.Toss(nodeId, nice, false, doSeen, noFile, noFreq, noExec, noTrns) || bad + bad = !ctx.Toss( + nodeId, TRx, nice, false, + doSeen, noFile, noFreq, noExec, noTrns, noArea) || bad } }() return finish, badCode diff --git a/src/toss_test.go b/src/toss_test.go index a66f43b..deb70f1 100644 --- a/src/toss_test.go +++ b/src/toss_test.go @@ -100,6 +100,7 @@ func TestTossExec(t *testing.T) { 1<<15, false, false, + nil, ); err != nil { t.Error(err) return false @@ -112,13 +113,15 @@ func TestTossExec(t *testing.T) { if len(dirFiles(rxPath)) == 0 { continue } - ctx.Toss(ctx.Self.Id, DefaultNiceExec-1, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec-1, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].Exec = make(map[string][]string) ctx.Neigh[*nodeOur.Id].Exec[handle] = []string{"/bin/sh", "-c", "false"} - ctx.Toss(ctx.Self.Id, DefaultNiceExec, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } @@ -130,7 +133,8 @@ func TestTossExec(t *testing.T) { filepath.Join(spool, "mbox"), ), } - ctx.Toss(ctx.Self.Id, DefaultNiceExec, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) != 0 { return false } @@ -204,6 +208,7 @@ func TestTossFile(t *testing.T) { MaxFileSize, 1<<15, MaxFileSize, + nil, ); err != nil { t.Error(err) return false @@ -211,12 +216,14 @@ func TestTossFile(t *testing.T) { } rxPath := filepath.Join(spool, ctx.Self.Id.String(), string(TRx)) os.Rename(filepath.Join(spool, ctx.Self.Id.String(), string(TTx)), rxPath) - ctx.Toss(ctx.Self.Id, DefaultNiceFile, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].Incoming = &incomingPath - ctx.Toss(ctx.Self.Id, DefaultNiceFile, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) != 0 { return false } @@ -281,6 +288,7 @@ func TestTossFileSameName(t *testing.T) { MaxFileSize, 1<<15, MaxFileSize, + nil, ); err != nil { t.Error(err) return false @@ -289,7 +297,8 @@ func TestTossFileSameName(t *testing.T) { rxPath := filepath.Join(spool, ctx.Self.Id.String(), string(TRx)) os.Rename(filepath.Join(spool, ctx.Self.Id.String(), string(TTx)), rxPath) ctx.Neigh[*nodeOur.Id].Incoming = &incomingPath - ctx.Toss(ctx.Self.Id, DefaultNiceFile, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, + false, false, false, false, false, false, false) expected := make(map[string]struct{}) expected["samefile"] = struct{}{} for i := 0; i < files-1; i++ { @@ -360,12 +369,14 @@ func TestTossFreq(t *testing.T) { txPath := filepath.Join(spool, ctx.Self.Id.String(), string(TTx)) os.Rename(txPath, rxPath) os.MkdirAll(txPath, os.FileMode(0700)) - ctx.Toss(ctx.Self.Id, DefaultNiceFreq, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, + false, false, false, false, false, false, false) if len(dirFiles(txPath)) != 0 || len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].FreqPath = &spool - ctx.Toss(ctx.Self.Id, DefaultNiceFreq, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, + false, false, false, false, false, false, false) if len(dirFiles(txPath)) != 0 || len(dirFiles(rxPath)) == 0 { return false } @@ -378,7 +389,8 @@ func TestTossFreq(t *testing.T) { panic(err) } } - ctx.Toss(ctx.Self.Id, DefaultNiceFreq, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, + false, false, false, false, false, false, false) if len(dirFiles(txPath)) == 0 || len(dirFiles(rxPath)) != 0 { return false } @@ -389,7 +401,7 @@ func TestTossFreq(t *testing.T) { t.Error(err) return false } - _, _, err = PktEncRead(ctx.Self, ctx.Neigh, fd, &buf) + _, _, _, err = PktEncRead(ctx.Self, ctx.Neigh, fd, &buf, true, nil) if err != nil { t.Error(err) return false @@ -483,7 +495,8 @@ func TestTossTrns(t *testing.T) { panic(err) } } - ctx.Toss(ctx.Self.Id, 123, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, 123, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) != 0 { return false } diff --git a/src/tx.go b/src/tx.go index 15cf08a..a90a49e 100644 --- a/src/tx.go +++ b/src/tx.go @@ -36,6 +36,7 @@ import ( xdr "github.com/davecgh/go-xdr/xdr2" "github.com/dustin/go-humanize" "github.com/klauspost/compress/zstd" + "golang.org/x/crypto/blake2b" "golang.org/x/crypto/chacha20poly1305" ) @@ -53,7 +54,15 @@ func (ctx *Ctx) Tx( size, minSize int64, src io.Reader, pktName string, + areaId *AreaId, ) (*Node, error) { + var area *Area + if areaId != nil { + area = ctx.AreaId2Area[*areaId] + if area.Prv == nil { + return nil, errors.New("unknown area id") + } + } hops := make([]*Node, 0, 1+len(node.Via)) hops = append(hops, node) lastNode := node @@ -62,7 +71,11 @@ func (ctx *Ctx) Tx( hops = append(hops, lastNode) } expectedSize := size - for i := 0; i < len(hops); i++ { + wrappers := len(hops) + if area != nil { + wrappers++ + } + for i := 0; i < wrappers; i++ { expectedSize = PktEncOverhead + PktSizeOverhead + sizeWithTags(PktOverhead+expectedSize) @@ -83,32 +96,84 @@ func (ctx *Ctx) Tx( pktEncRaws := make(chan []byte) curSize := size pipeR, pipeW := io.Pipe() - go func(size int64, src io.Reader, dst io.WriteCloser) { - ctx.LogD("tx", LEs{ - {"Node", hops[0].Id}, - {"Nice", int(nice)}, - {"Size", size}, - }, func(les LEs) string { - return fmt.Sprintf( - "Tx packet to %s (%s) nice: %s", - ctx.NodeName(hops[0].Id), - humanize.IBytes(uint64(size)), - NicenessFmt(nice), - ) - }) - pktEncRaw, err := PktEncWrite( - ctx.Self, hops[0], pkt, nice, size, padSize, src, dst, - ) - pktEncRaws <- pktEncRaw - errs <- err - dst.Close() // #nosec G104 - }(curSize, src, pipeW) - curSize = PktEncOverhead + - PktSizeOverhead + - sizeWithTags(PktOverhead+curSize) + - padSize - var pipeRPrev io.Reader + if area == nil { + go func(size int64, src io.Reader, dst io.WriteCloser) { + ctx.LogD("tx", LEs{ + {"Node", hops[0].Id}, + {"Nice", int(nice)}, + {"Size", size}, + }, func(les LEs) string { + return fmt.Sprintf( + "Tx packet to %s (%s) nice: %s", + ctx.NodeName(hops[0].Id), + humanize.IBytes(uint64(size)), + NicenessFmt(nice), + ) + }) + pktEncRaw, err := PktEncWrite( + ctx.Self, hops[0], pkt, nice, size, padSize, src, dst, + ) + pktEncRaws <- pktEncRaw + errs <- err + dst.Close() // #nosec G104 + }(curSize, src, pipeW) + curSize = PktEncOverhead + PktSizeOverhead + sizeWithTags(PktOverhead+curSize) + curSize += padSize + } else { + go func(size, padSize int64, src io.Reader, dst io.WriteCloser) { + ctx.LogD("tx", LEs{ + {"Area", area.Id}, + {"Nice", int(nice)}, + {"Size", size}, + }, func(les LEs) string { + return fmt.Sprintf( + "Tx area packet to %s (%s) nice: %s", + ctx.AreaName(areaId), + humanize.IBytes(uint64(size)), + NicenessFmt(nice), + ) + }) + areaNode := Node{Id: new(NodeId), ExchPub: new([32]byte)} + copy(areaNode.Id[:], area.Id[:]) + copy(areaNode.ExchPub[:], area.Pub[:]) + pktEncRaw, err := PktEncWrite( + ctx.Self, &areaNode, pkt, nice, size, padSize, src, dst, + ) + pktEncRaws <- pktEncRaw + errs <- err + dst.Close() // #nosec G104 + }(curSize, padSize, src, pipeW) + curSize = PktEncOverhead + PktSizeOverhead + sizeWithTags(PktOverhead+curSize) + curSize += padSize + pipeRPrev = pipeR + pipeR, pipeW = io.Pipe() + go func(size int64, src io.Reader, dst io.WriteCloser) { + pktArea, err := NewPkt(PktTypeArea, 0, area.Id[:]) + if err != nil { + panic(err) + } + ctx.LogD("tx", LEs{ + {"Node", hops[0].Id}, + {"Nice", int(nice)}, + {"Size", size}, + }, func(les LEs) string { + return fmt.Sprintf( + "Tx packet to %s (%s) nice: %s", + ctx.NodeName(hops[0].Id), + humanize.IBytes(uint64(size)), + NicenessFmt(nice), + ) + }) + pktEncRaw, err := PktEncWrite( + ctx.Self, hops[0], pktArea, nice, size, 0, src, dst, + ) + pktEncRaws <- pktEncRaw + errs <- err + dst.Close() // #nosec G104 + }(curSize, pipeRPrev, pipeW) + curSize = PktEncOverhead + PktSizeOverhead + sizeWithTags(PktOverhead+curSize) + } for i := 1; i < len(hops); i++ { pktTrns, err := NewPkt(PktTypeTrns, 0, hops[i-1].Id[:]) if err != nil { @@ -145,10 +210,14 @@ func (ctx *Ctx) Tx( errs <- err }() var pktEncRaw []byte + var pktEncMsg []byte + if area != nil { + pktEncMsg = <-pktEncRaws + } for i := 0; i < len(hops); i++ { pktEncRaw = <-pktEncRaws } - for i := 0; i <= len(hops); i++ { + for i := 0; i <= wrappers; i++ { err = <-errs if err != nil { tmp.Fd.Close() // #nosec G104 @@ -164,6 +233,43 @@ func (ctx *Ctx) Tx( if ctx.HdrUsage { ctx.HdrWrite(pktEncRaw, filepath.Join(nodePath, string(TTx), tmp.Checksum())) } + if area != nil { + msgHashRaw := blake2b.Sum256(pktEncMsg) + msgHash := Base32Codec.EncodeToString(msgHashRaw[:]) + seenDir := filepath.Join( + ctx.Spool, ctx.SelfId.String(), AreaDir, areaId.String(), + ) + seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + les := LEs{ + {"Node", node.Id}, + {"Nice", int(nice)}, + {"Size", size}, + {"Area", areaId}, + {"AreaMsg", msgHash}, + } + logMsg := func(les LEs) string { + return fmt.Sprintf( + "Tx area packet to %s (%s) nice: %s, area %s: %s", + ctx.NodeName(node.Id), + humanize.IBytes(uint64(size)), + NicenessFmt(nice), + area.Name, + msgHash, + ) + } + if err = os.MkdirAll(seenDir, os.FileMode(0777)); err != nil { + ctx.LogE("tx-mkdir", les, err, logMsg) + return lastNode, err + } + if fd, err := os.Create(seenPath); err == nil { + fd.Close() + if err = DirSync(seenDir); err != nil { + ctx.LogE("tx-dirsync", les, err, logMsg) + return lastNode, err + } + } + ctx.LogI("tx-area", les, logMsg) + } return lastNode, err } @@ -353,6 +459,7 @@ func (ctx *Ctx) TxFile( srcPath, dstPath string, chunkSize int64, minSize, maxSize int64, + areaId *AreaId, ) error { dstPathSpecified := false if dstPath == "" { @@ -386,7 +493,7 @@ func (ctx *Ctx) TxFile( if err != nil { return err } - _, err = ctx.Tx(node, pkt, nice, fileSize, minSize, reader, dstPath) + _, err = ctx.Tx(node, pkt, nice, fileSize, minSize, reader, dstPath, areaId) les := LEs{ {"Type", "file"}, {"Node", node.Id}, @@ -448,6 +555,7 @@ func (ctx *Ctx) TxFile( minSize, io.TeeReader(reader, hsh), path, + areaId, ) les := LEs{ {"Type", "file"}, @@ -490,7 +598,7 @@ func (ctx *Ctx) TxFile( return err } metaPktSize := int64(metaBuf.Len()) - _, err = ctx.Tx(node, pkt, nice, metaPktSize, minSize, &metaBuf, path) + _, err = ctx.Tx(node, pkt, nice, metaPktSize, minSize, &metaBuf, path, areaId) les := LEs{ {"Type", "file"}, {"Node", node.Id}, @@ -535,7 +643,7 @@ func (ctx *Ctx) TxFreq( } src := strings.NewReader(dstPath) size := int64(src.Len()) - _, err = ctx.Tx(node, pkt, nice, size, minSize, src, srcPath) + _, err = ctx.Tx(node, pkt, nice, size, minSize, src, srcPath, nil) les := LEs{ {"Type", "freq"}, {"Node", node.Id}, @@ -568,6 +676,7 @@ func (ctx *Ctx) TxExec( minSize int64, useTmp bool, noCompress bool, + areaId *AreaId, ) error { path := make([][]byte, 0, 1+len(args)) path = append(path, []byte(handle)) @@ -601,7 +710,7 @@ func (ctx *Ctx) TxExec( return err } size = int64(compressed.Len()) - _, err = ctx.Tx(node, pkt, nice, size, minSize, &compressed, handle) + _, err = ctx.Tx(node, pkt, nice, size, minSize, &compressed, handle, areaId) } if noCompress && !useTmp { var data bytes.Buffer @@ -609,7 +718,7 @@ func (ctx *Ctx) TxExec( return err } size = int64(data.Len()) - _, err = ctx.Tx(node, pkt, nice, size, minSize, &data, handle) + _, err = ctx.Tx(node, pkt, nice, size, minSize, &data, handle, areaId) } if !noCompress && useTmp { r, w := io.Pipe() @@ -640,7 +749,7 @@ func (ctx *Ctx) TxExec( return err } size = fileSize - _, err = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle) + _, err = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle, areaId) } if noCompress && useTmp { tmpReader, closer, fileSize, err := throughTmpFile(in) @@ -651,7 +760,7 @@ func (ctx *Ctx) TxExec( return err } size = fileSize - _, err = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle) + _, err = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle, areaId) } dst := strings.Join(append([]string{handle}, args...), " ") diff --git a/src/tx_test.go b/src/tx_test.go index 72c6f61..85a00db 100644 --- a/src/tx_test.go +++ b/src/tx_test.go @@ -84,6 +84,7 @@ func TestTx(t *testing.T) { int64(padSize), src, "pktName", + nil, ) if err != nil { return false @@ -110,7 +111,9 @@ func TestTx(t *testing.T) { vias := append(nodeTgt.Via, nodeTgt.Id) for i, hopId := range vias { hopOur := privates[*hopId] - foundNode, _, err := PktEncRead(hopOur, ctx.Neigh, &bufR, &bufW) + _, foundNode, _, err := PktEncRead( + hopOur, ctx.Neigh, &bufR, &bufW, true, nil, + ) if err != nil { return false } -- 2.44.0