From: Sergey Matveev Date: Sun, 4 Jul 2021 17:27:24 +0000 (+0300) Subject: Merge branch 'develop' X-Git-Tag: v7.1.0^0 X-Git-Url: http://www.git.cypherpunks.ru/?a=commitdiff_plain;h=0ee0ba3784ce20e69d8db6316c99e7f0b6d30809;hp=1cc0df98a8d949b9f8137081b875d98a1aae2e67;p=nncp.git Merge branch 'develop' --- diff --git a/README b/README index eca5b5a..9aa5a78 100644 --- a/README +++ b/README @@ -8,7 +8,7 @@ requests, Internet mail and commands transmission. All packets are integrity checked, end-to-end encrypted (E2EE), explicitly authenticated by known participants public keys. Onion encryption is applied to relayed packets. Each node acts both as a client and server, can use -push and poll behaviour model. +push and poll behaviour model. Also there is multicasting areas support. Out-of-box offline sneakernet/floppynet, dead drops, sequential and append-only CD-ROM/tape storages, air-gapped computers support. But diff --git a/README.RU b/README.RU index 9da6599..a48bc31 100644 --- a/README.RU +++ b/README.RU @@ -10,7 +10,7 @@ NNCP (Node to Node copy) это набор утилит упрощающий б ключами участников. Луковичное (onion) шифрование применяется ко всем ретранслируемым пакетам. Каждый узел выступает одновременно в роли клиента и сервера, может использовать как push, так и poll модель -поведения. +поведения. А также есть поддержка мультивещательной рассылки пакетов. Поддержка из коробки offline флоппинета, тайников для сброса информации (dead drop), последовательных и не перезаписываемых CD-ROM/ленточных diff --git a/doc/about.ru.texi b/doc/about.ru.texi index bb9c2d0..bd47b1a 100644 --- a/doc/about.ru.texi +++ b/doc/about.ru.texi @@ -20,6 +20,7 @@ (onion) шифрование применяется ко всем ретранслируемым пакетам. Каждый узел выступает одновременно в роли клиента и сервера, может использовать как push, так и poll модель поведения. +А также есть поддержка @ref{Multicast, мультивещательной} рассылки пакетов. Поддержка из коробки offline @url{https://ru.wikipedia.org/wiki/%D0%A4%D0%BB%D0%BE%D0%BF%D0%BF%D0%B8%D0%BD%D0%B5%D1%82, diff --git a/doc/about.texi b/doc/about.texi index f228d15..f752931 100644 --- a/doc/about.texi +++ b/doc/about.texi @@ -15,6 +15,7 @@ end-to-end} encrypted, explicitly authenticated by known participants public keys. @url{https://en.wikipedia.org/wiki/Onion_routing, Onion encryption} is applied to relayed packets. Each node acts both as a client and server, can use push and poll behaviour model. +Also there is @ref{Multicast, multicasting} areas support. Out-of-box offline @url{https://en.wikipedia.org/wiki/Sneakernet, sneakernet/floppynet}, @url{https://en.wikipedia.org/wiki/Dead_drop, diff --git a/doc/call.texi b/doc/call.texi index 86f4bcf..83f43ea 100644 --- a/doc/call.texi +++ b/doc/call.texi @@ -1,7 +1,7 @@ @node Call @unnumbered Call configuration -Call is a rule when and how node can be called. +Call is a rule when and how node can be called by @ref{nncp-caller}. Example list of call structures: diff --git a/doc/cfg.texi b/doc/cfg.texi deleted file mode 100644 index 30341f8..0000000 --- a/doc/cfg.texi +++ /dev/null @@ -1,239 +0,0 @@ -@node Configuration -@unnumbered Configuration file - -Example @url{https://hjson.org/, Hjson} configuration file: - -@verbatim -{ - spool: /var/spool/nncp - log: /var/spool/nncp/log - umask: "022" - noprogress: true - nohdr: true - - mcd-listen: ["em0", "igb1"] - mcd-send: {em0: 60, igb1: 5} - - notify: { - file: { - from: nncp@localhost - to: user+file@example.com - } - freq: { - from: nncp@localhost - to: user+freq@example.com - } - exec: { - "*.warcer": { - from: nncp@localhost - to: user+warcer@example.com - } - "eve.warcer": { - from: nncp@localhost - to: user+warcer-overriden@example.com - } - } - } - - self: { - id: TIJQL...2NGIA - exchpub: CYVGQ...PSEWQ - exchprv: 65PUY...MPZ3Q - signpub: 2NMVC...CMH5Q - signprv: 555JD...RGD6Y - noiseprv: D62XU...NKYPA - noisepub: KIBKK...ESM7Q - } - - neigh: { - self: { - id: TIJQL...2NGIA - exchpub: CYVGQ...PSEWQ - signpub: 2NMVC...CMH5Q - noisepub: KIBKK...ESM7Q - exec: {sendmail: ["/usr/sbin/sendmail"]} - } - alice: { - id: "XJZBK...65IJQ" - exchpub: MJACJ...FAI6A - signpub: T4AFC...N2FRQ - noisepub: UBM5K...VI42A - exec: {flag: ["/usr/bin/touch", "-t"]} - incoming: "/home/alice/incoming" - onlinedeadline: 1800 - maxonlinetime: 3600 - addrs: { - lan: "[fe80::1234%igb0]:5400" - internet: alice.com:3389 - proxied: "|ssh remote.host nncp-daemon -inetd" - } - calls: [ - { - cron: "*/2 * * * *" - }, - ] - } - bob: { - id: 2IZNP...UYGYA - exchpub: WFLMZ...B7NHA - signpub: GTGXG...IE3OA - exec: { - sendmail: ["/usr/sbin/sendmail"] - warcer: ["/path/to/warcer.sh"] - wgeter: ["/path/to/wgeter.sh"] - } - freq: { - path: "/home/bob/pub" - chunked: 1024 - minsize: 2048 - } - via: ["alice"] - rxrate: 10 - txrate: 20 - } - } -} -@end verbatim - -@strong{spool} field contains an absolute path to @ref{Spool, spool} -directory. @strong{log} field contains an absolute path to @ref{Log, -log} file. - -Non-empty optional @strong{umask} will force all invoked commands to -override their umask to specified octal mask. Useful for using with -@ref{Shared spool, shared spool directories}. - -Enabled @strong{noprogress} option disabled progress showing for many -commands by default. You can always force its showing with -@option{-progress} command line option anyway. - -@anchor{CfgNoHdr} -@strong{nohdr} option disables @ref{HdrFile, .hdr} files usage. - -@anchor{CfgMCDListen} -@strong{mcd-listen} specifies list of network interfaces -@ref{nncp-caller} will listen for incoming @ref{MCD} announcements. - -@anchor{CfgMCDSend} -@strong{mcd-send} specifies list of network interfaces, and intervals in -seconds, where @ref{nncp-daemon} will send @ref{MCD} announcements. - -@anchor{CfgNotify} -@strong{notify} section contains notification settings for successfully -tossed file, freq and exec packets. Corresponding @strong{from} and -@strong{to} fields will be substituted in notification email message. -@code{neigh.self.exec.sendmail} will be used as a local mailer. You can -omit either of those two @code{from}/@code{to} sections to omit -corresponding notifications, or the whole section at once. - -@code{notify.exec} section is a mapping of exec handles and -corresponding @code{from}/@code{to} sections. Each handle has either -@code{NODE.HANDLE} or @code{*.HANDLE} syntax. You can override -notification options for some node with the first type of name. -Handle command's output will be included in notification messages. - -@strong{self} section contains our node's private keypairs. -@strong{exch*} and @strong{sign*} are used during @ref{Encrypted, -encrypted} packet creation. @strong{noise*} are used during @ref{Sync, -synchronization protocol} working in @ref{nncp-call}/@ref{nncp-daemon}. - -@strong{neigh} section contains all known neighbours information. It -always has @strong{self} neighbour that is copy of our node's public -data (public keys). It is useful for copy-paste sharing with your -friends. Each section's key is a human-readable name of the neighbour. - -Except for @code{id}, @code{exchpub} and @code{signpub} each neighbour -node has the following fields: - -@table @strong - -@item noisepub -If present, then node can be online called using @ref{Sync, -synchronization protocol}. Contains authentication public key. - -@anchor{CfgExec} -@item exec -Dictionary consisting of handles and corresponding command line -arguments. In example above there are @command{sendmail} handles, -@command{warcer}, @command{wgeter} and @command{flag} one. Remote node -can queue some handle execution with providing additional command line -arguments and the body fed to command's @code{stdin}. - -@verb{|sendmail: ["/usr/sbin/sendmail", "-t"]|} handle, when called by -@verb{|echo hello world | nncp-exec OURNODE sendmail ARG0 ARG1 ARG2|} -command, will execute: - -@example -NNCP_SELF=OURNODE \ -NNCP_SENDER=REMOTE \ -NNCP_NICE=64 \ -/usr/sbin/sendmail -t ARG0 ARG1 ARG2 -@end example - -feeding @verb{|hello world\n|} to that started @command{sendmail} -process. - -@anchor{CfgIncoming} -@item incoming -Full path to directory where all file uploads will be saved. May be -omitted to forbid file uploading on that node. - -@anchor{CfgFreq} -@item freq.path -Full path to directory from where file requests will queue files for -transmission. May be omitted to forbid freqing from that node. - -@item freq.chunked -If set, then enable @ref{Chunked, chunked} file transmission during -freqing. This is the desired chunk size in KiBs. - -@item freq.minsize -If set, then apply @ref{OptMinSize, -minsize} option during file -transmission. - -@anchor{CfgVia} -@item via -An array of node identifiers that will be used as a relay to that node. -For example @verb{|["foo","bar"]|} means that packet can reach current -node by transitioning through @code{foo} and then @code{bar} nodes. May -be omitted if direct connection exists and no relaying is required. - -@anchor{CfgAddrs} -@item addrs -Dictionary containing known network addresses of the node. Each key is -human-readable name of the address. For direct TCP connections use -@verb{|host:port|} format, pointing to @ref{nncp-daemon}'s listening -instance. Also you can pipe connection through the external command -using @verb{#|some command#} format. @code{/bin/sh -c "some command"} -will start and its @code{stdin}/@code{stdout} used as a connection. May -be omitted if either no direct connection exists, or @ref{nncp-call} is -used with forced address specifying. - -@anchor{CfgXxRate} -@item rxrate/txrate -If greater than zero, then at most *rate packets per second will be -sent/received after the handshake. It could be used as crude bandwidth -traffic shaper: each packet has at most 64 KiB payload size. Could be -omitted at all -- no rate limits. - -@anchor{CfgOnlineDeadline} -@item onlinedeadline -Online connection deadline of nodes inactivity in seconds. It is the -time connection considered dead after not receiving/sending any packets -(except for PINGs) and connection must be terminated. By default it is -set to 10 seconds. This can be set to rather high values to keep -connection alive (to reduce handshake overhead and delays), wait for -appearing packets ready to send and notifying remote side about their -appearance. - -@anchor{CfgMaxOnlineTime} -@item maxonlinetime -If greater than zero, then it is maximal time of single connection. -Forcefully disconnect if it is exceeded. - -@anchor{CfgCalls} -@item calls -List of @ref{Call, call configuration}s. Can be omitted if -@ref{nncp-caller} won't be used to call that node. - -@end table diff --git a/doc/cfg/areas.texi b/doc/cfg/areas.texi new file mode 100644 index 0000000..bc21008 --- /dev/null +++ b/doc/cfg/areas.texi @@ -0,0 +1,67 @@ +@node CfgAreas +@section Configuration areas options + +@ref{Multicast} areas configuration only used with multicast packets. + +@verbatim +areas: { + nodelist: { + id: OU67K7NA3RPOPFKJWNVBYJ5GPLRBDGHH6DZSSJ32JL7Q3Q76E52A + + pub: ALCX2NJBANMBNFTQ27C3C6W2WJIXSE74R27TSYZQKMD2UJERCEOQ + prv: VQ3B4TLAZZB2G7RS3OSS5NUVKAS44OGY5YMQPMTAHQMZZLNG25MA + + subs: ["alice", "bob", "eve"] + incoming: /home/incoming/areas/nodelist + } + echoarea: { + id: CKKJ3HOAVOP7VPNCEGZRNDO34MUOOJ4AXHDFCSVSOE647KN5CMIA + + pub: 5MFPTJI2R322EUCTGCWZXTDBCVEL5NCFDBXI5PHPQOTLUVSQ3ZIQ + prv: LVGIZQRQTDE524KEE5FOWLE2GCQBILY4VSQBDHWJC6YUTOJ54QCQ + + subs: ["alice", "bob"] + exec: {sendmail: ["/usr/sbin/sendmail"]} + allow-unknown: true + } + whatever.pvt: { + id: OU67K7NA3RPOPFKJWNVBYJ5GPLRBDGHH6DZSSJ32JL7Q3Q76E52A + subs: ["dave", "eve"] + } +} +@end verbatim + +Each key is human readable multicast group/area/echo name. + +The only required field is the @code{id}. You can not process multicast +packets that has unknown area identification. + +@code{subs} contains a list of recipients you must relay incoming +multicast packet on. + +Knowledge of @code{pub} and @code{prv} keys gives ability to decrypt +multicast packet and process its contents (file or exec transmission). +For accepting file transmissions you must set @code{incoming}, similar +to @ref{CfgIncoming, neigh's node option}. For accepting exec +transmissions you must set @code{exec}, similar to @ref{CfgExec, neigh's +node option}. + +You can accept multicast packets from unknown senders, by setting +@code{allow-unknown} option. + +In the example above: + +@table @code +@item nodelist +That area is for multicast sending of @file{nodelist} files, with +relaying it to @code{alice}, @code{bob} and @code{eve} further. +@item echoarea +That area is for multicast discussion through @code{sendmail} handled +exec packets. Relaying to @code{alice} and @code{bob} and accepting +messages from unknown participants. +@item whatever.pvt +We just relay that area packets to @code{dave} and @code{eve}, but +without ability to see what is inside them. Pay attention that +@code{allow-unknown} does not play any role here, because we are not +even trying to decrypt (and authenticate) those multicast packets. +@end table diff --git a/doc/cfg/general.texi b/doc/cfg/general.texi new file mode 100644 index 0000000..63d0757 --- /dev/null +++ b/doc/cfg/general.texi @@ -0,0 +1,48 @@ +@node CfgGeneral +@section Configuration general options + +Those options are in the root of configuration dictionary. + +@verbatim +spool: /var/spool/nncp +log: /var/spool/nncp/log + +# All of options below are optional +umask: "022" +noprogress: true +nohdr: true + +# MultiCast Discovery +mcd-listen: ["em0", "igb1"] +mcd-send: {em0: 60, igb1: 5} +@end verbatim + +@table @code +@item spool +Absolute path to the @ref{Spool, spool} directory. +@item log +Absolute path to the @ref{Log, log} file. +@item umask +Will force all invoked commands to override their umask to specified +octal mask. Useful for using with @ref{Shared spool, shared spool directories}. +@item noprogress +When enabled, disables progress showing for many commands by default. +You can always force its showing with @option{-progress} command line +option anyway. +@anchor{CfgNoHdr} +@item nohdr +@strong{nohdr} option disables @ref{HdrFile, .hdr} files usage. +@end table + +And optional @ref{MCD, MultiCast Discovery} options: + +@table @code +@anchor{CfgMCDListen} +@item mcd-listen +Specifies list of network interfaces @ref{nncp-caller} will listen for +incoming @ref{MCD} announcements. +@anchor{CfgMCDSend} +@item mcd-send +Specifies list of network interfaces, and intervals in seconds, where +@ref{nncp-daemon} will send @ref{MCD} announcements. +@end table diff --git a/doc/cfg/index.texi b/doc/cfg/index.texi new file mode 100644 index 0000000..1944eaa --- /dev/null +++ b/doc/cfg/index.texi @@ -0,0 +1,49 @@ +@node Configuration +@unnumbered Configuration file + +NNCP uses single file configuration file in @url{https://hjson.org/, +Hjson} format. Initially it is created with @ref{nncp-cfgnew} command +and at minimum it can look like this: + +@verbatim +spool: /var/spool/nncp +log: /var/spool/nncp/log + +self: { + id: RKOLY...KAMXQ + exchpub: 2NZKH...CMI7A + exchprv: KETPP...2OJZA + signpub: EXD7M...YAOFA + signprv: B3EMS..XMAHCQ + noiseprv: 3TJDF...2D7DQ + noisepub: MIXYN...BGNDQ +} + +neigh: { + self: { + id: RKOLY...KAMXQ + exchpub: 2NZKH...CMI7A + signpub: EXD7M...YAOFA + noisepub: MIXYN...BGNDQ + } +} +@end verbatim + +And for being able to communicate with at least one other node, you just +need to add single key to the @code{neigh} section similar to the "self". + +All configuration file can be separated on five sections: + +@menu +* General options: CfgGeneral. +* Self-node keypairs: CfgSelf. +* Notifications: CfgNotify. +* Neighbours: CfgNeigh. +* Areas: CfgAreas. +@end menu + +@include cfg/general.texi +@include cfg/self.texi +@include cfg/notify.texi +@include cfg/neigh.texi +@include cfg/areas.texi diff --git a/doc/cfg/neigh.texi b/doc/cfg/neigh.texi new file mode 100644 index 0000000..53eed46 --- /dev/null +++ b/doc/cfg/neigh.texi @@ -0,0 +1,160 @@ +@node CfgNeigh +@section Configuration neighbour options + +@strong{neigh} section contains all known neighbours information. It +always has @strong{self} neighbour that is copy of our node's public +data (public keys). It is useful for copy-paste sharing with your +friends. Each section's key is a human-readable name of the neighbour. + +@verbatim +neigh: { + self: { + id: RKOLY...KAMXQ + exchpub: 2NZKH...CMI7A + signpub: EXD7M...YAOFA + noisepub: MIXYN...BGNDQ + exec: {sendmail: ["/usr/sbin/sendmail"]} + } + alice: { + id: "XJZBK...65IJQ" + exchpub: MJACJ...FAI6A + signpub: T4AFC...N2FRQ + noisepub: UBM5K...VI42A + exec: {flag: ["/usr/bin/touch", "-t"]} + incoming: "/home/alice/incoming" + onlinedeadline: 1800 + maxonlinetime: 3600 + addrs: { + lan: "[fe80::1234%igb0]:5400" + internet: alice.com:3389 + proxied: "|ssh remote.host nncp-daemon -inetd" + } + calls: [ + { + cron: "*/2 * * * *" + }, + ] + } + bob: { + id: 2IZNP...UYGYA + exchpub: WFLMZ...B7NHA + signpub: GTGXG...IE3OA + exec: { + sendmail: ["/usr/sbin/sendmail"] + warcer: ["/path/to/warcer.sh"] + wgeter: ["/path/to/wgeter.sh"] + } + freq: { + path: "/home/bob/pub" + chunked: 1024 + minsize: 2048 + } + via: ["alice"] + rxrate: 10 + txrate: 20 + } +} +@end verbatim + +Except for @code{id}, @code{exchpub} and @code{signpub} each neighbour +node has the following fields: + +@table @code + +@item noisepub + If present, then node can be online called using @ref{Sync, + synchronization protocol}. Contains authentication public key. + +@anchor{CfgExec} + @item exec + Dictionary consisting of handles and corresponding command line + arguments. In example above there are @command{sendmail} handles, + @command{warcer}, @command{wgeter} and @command{flag} one. Remote + node can queue some handle execution with providing additional + command line arguments and the body fed to command's @code{stdin}. + + @verb{|sendmail: ["/usr/sbin/sendmail", "-t"]|} handle, when called by + @verb{|echo hello world | nncp-exec self sendmail ARG0 ARG1 ARG2|} + command, will execute: + +@example +NNCP_SELF=OURNODE \ +NNCP_SENDER=REMOTE \ +NNCP_NICE=64 \ +/usr/sbin/sendmail -t ARG0 ARG1 ARG2 +@end example + + feeding @verb{|hello world\n|} to that started @command{sendmail} + process. + +@anchor{CfgIncoming} +@item incoming + Full path to directory where all file uploads will be saved. May be + omitted to forbid file uploading on that node. + +@anchor{CfgFreq} +@item freq + @table @code + @item path + Full path to directory from where file requests will queue files + for transmission. May be omitted to forbid freqing from that node. + + @item chunked + If set, then enable @ref{Chunked, chunked} file transmission + during freqing. This is the desired chunk size in KiBs. + + @item minsize + If set, then apply @ref{OptMinSize, -minsize} option during file + transmission. + @end table + +@anchor{CfgVia} +@item via + An array of node identifiers that will be used as a relay to that + node. For example @verb{|["foo","bar"]|} means that packet can reach + current node by transitioning through @code{foo} and then @code{bar} + nodes. May be omitted if direct connection exists and no relaying is + required. + +@anchor{CfgAddrs} +@item addrs + Dictionary containing known network addresses of the node. Each key + is human-readable name of the address. For direct TCP connections + use @verb{|host:port|} format, pointing to @ref{nncp-daemon}'s + listening instance. + + Also you can pipe connection through the external command using + @verb{#|some command#} format. @code{/bin/sh -c "some command"} will + start and its @code{stdin}/@code{stdout} used as a connection. + + May be omitted if either no direct connection exists, or + @ref{nncp-call} is used with forced address specifying. + +@anchor{CfgXxRate} +@item rxrate/txrate + If greater than zero, then at most *rate packets per second will be + sent/received after the handshake. It could be used as crude + bandwidth traffic shaper: each packet has at most 64 KiB payload + size. If omitted -- no rate limits. + +@anchor{CfgOnlineDeadline} +@item onlinedeadline + Online connection deadline of nodes inactivity in seconds. It is the + time connection considered dead after not receiving/sending any + packets (except for PINGs) and connection must be terminated. By + default it is set to 10 seconds. This can be set to rather high + values to keep connection alive (to reduce handshake overhead and + delays), wait for appearing packets ready to send and notifying + remote side about their appearance. + +@anchor{CfgMaxOnlineTime} +@item maxonlinetime + If greater than zero, then it is maximal time of single connection. + Forcefully disconnect if it is exceeded. + +@anchor{CfgCalls} +@item calls + List of @ref{Call, call configuration}s. + Can be omitted if @ref{nncp-caller} won't be used to call that node. + +@end table diff --git a/doc/cfg/notify.texi b/doc/cfg/notify.texi new file mode 100644 index 0000000..8badc94 --- /dev/null +++ b/doc/cfg/notify.texi @@ -0,0 +1,42 @@ +@node CfgNotify +@section Configuration notification options + +That section controls what notifications are enabled and how must be +sent through the email. Notifications are sent for successful tossing of +file, freq or exec packet. + +@verbatim +notify: { + file: { + from: nncp@localhost + to: user+file@example.com + } + freq: { + from: nncp@localhost + to: user+freq@example.com + } + exec: { + bob.somehandle: { + from: nncp+bob@localhost + to: user+somehandle@example.com + } + *.anotherhandle: { + from: nncp@localhost + to: user+anotherhandle@example.com + } + } +} +@end verbatim + +Corresponding @strong{from} and @strong{to} fields will be substituted +in notification email message. @code{neigh.self.exec.sendmail} will be +used as a local mailer (command called for sending email message). + +You can omit either of those two @code{from}/@code{to} sections to omit +corresponding notifications, or the whole section at once. + +@code{notify.exec} section is a mapping of exec handles and +corresponding @code{from}/@code{to} sections. Each handle has either +@code{NODE.HANDLE} or @code{*.HANDLE} syntax. You can override +notification options for some node with the first type of name. Handle +command's output will be included in notification messages. diff --git a/doc/cfg/self.texi b/doc/cfg/self.texi new file mode 100644 index 0000000..0528b5a --- /dev/null +++ b/doc/cfg/self.texi @@ -0,0 +1,10 @@ +@node CfgSelf +@section Configuration self-node keypairs + +@strong{self} section contains our node's private keypairs. + +@strong{exch*} and @strong{sign*} are used during @ref{Encrypted, +encrypted} packet creation. + +@strong{noise*} are used during @ref{Sync, synchronization protocol} +working in @ref{nncp-call}, @ref{nncp-caller}, @ref{nncp-daemon}. diff --git a/doc/cmd/index.texi b/doc/cmd/index.texi new file mode 100644 index 0000000..cb81543 --- /dev/null +++ b/doc/cmd/index.texi @@ -0,0 +1,104 @@ +@node Commands +@unnumbered Commands + +Nearly all commands have the following common options: + +@table @option +@item -cfg + Path to configuration file. May be overridden by @env{NNCPCFG} + environment variable. If file file is an encrypted @ref{EBlob, + eblob}, then ask for passphrase to decrypt it first. +@item -debug + Print debug messages. Normally this option should not be used. +@item -minsize + @anchor{OptMinSize} + Minimal required resulting packet size, in KiBs. For example if you + send 2 KiB file and set @option{-minsize 4}, then resulting packet + will be 4 KiB (containing file itself and some junk). +@item -nice + Set desired outgoing packet @ref{Niceness, niceness level}. +@item -replynice + Set desired reply packet @ref{Niceness, niceness level}. Only freq + and exec packets look at that niceness level. +@item -via + Override @ref{CfgVia, via} configuration option for destination node. + Specified nodes must be separated with comma: @verb{|NODE1,NODE2|}. + With @verb{|-via -|} you can disable relaying at all. +@item -spool + Override path to spool directory. May be specified by + @env{NNCPSPOOL} environment variable. +@item -log + Override path to logfile. May be specified by @env{NNCPLOG} + environment variable. +@item -quiet + Print only errors, omit simple informational messages. In any case + those messages are logged, so you can reread them using + @ref{nncp-log} command. +@item -progress, -noprogress + Either force progress showing, or disable it. +@item -version + Print version information. +@item -warranty + Print warranty information (no warranty). +@end table + +@menu +Configuration file commands + +* nncp-cfgnew:: +* nncp-cfgmin:: +* nncp-cfgenc:: + +Packets creation commands + +* nncp-file:: +* nncp-exec:: +* nncp-freq:: + +Packets sharing commands + +* nncp-xfer:: +* nncp-bundle:: + +Checking and tossing commands + +* nncp-toss:: +* nncp-check:: +* nncp-reass:: + +Online synchronization protocol commands + +* nncp-daemon:: +* nncp-call:: +* nncp-caller:: +* nncp-cronexpr:: + +Maintenance, monitoring and debugging commands: + +* nncp-stat:: +* nncp-log:: +* nncp-rm:: +* nncp-pkt:: +* nncp-hash:: +@end menu + +@include cmd/nncp-cfgnew.texi +@include cmd/nncp-cfgmin.texi +@include cmd/nncp-cfgenc.texi +@include cmd/nncp-file.texi +@include cmd/nncp-exec.texi +@include cmd/nncp-freq.texi +@include cmd/nncp-xfer.texi +@include cmd/nncp-bundle.texi +@include cmd/nncp-toss.texi +@include cmd/nncp-check.texi +@include cmd/nncp-reass.texi +@include cmd/nncp-daemon.texi +@include cmd/nncp-call.texi +@include cmd/nncp-caller.texi +@include cmd/nncp-cronexpr.texi +@include cmd/nncp-stat.texi +@include cmd/nncp-log.texi +@include cmd/nncp-rm.texi +@include cmd/nncp-pkt.texi +@include cmd/nncp-hash.texi diff --git a/doc/cmd/nncp-bundle.texi b/doc/cmd/nncp-bundle.texi new file mode 100644 index 0000000..9508946 --- /dev/null +++ b/doc/cmd/nncp-bundle.texi @@ -0,0 +1,46 @@ +@node nncp-bundle +@section nncp-bundle + +@example +$ nncp-bundle [options] -tx [-delete] NODE [NODE ...] > ... +$ nncp-bundle [options] -rx -delete [-dryrun] [NODE ...] < ... +$ nncp-bundle [options] -rx [-check] [-dryrun] [NODE ...] < ... +@end example + +With @option{-tx} option, this command creates @ref{Bundles, bundle} of +@ref{Encrypted, encrypted packets} from the spool directory and writes +it to @code{stdout}. + +With @option{-rx} option, this command takes bundle from @code{stdin} +and copies all found packets for our node to the spool directory. Pay +attention that @strong{no} integrity checking is done by default. Modern +tape drives could easily provide too much throughput your CPU won't be +able to verify on the fly. So if you won't @ref{nncp-toss, toss} +received packets at the place, it is advisable either to run +@ref{nncp-check} utility for packets integrity verification, or to use +@option{-check} option to enable on the fly integrity check. + +You can specify multiple @option{NODE} arguments, telling for what nodes +you want to create the stream, or take it from. If no nodes are +specified for @option{-rx} mode, then all packets aimed at us will be +processed. + +When packets are sent through the stream, they are still kept in the +spool directory, because there is no assurance that they are transferred +to the media (media (CD-ROM, tape drive, raw hard drive) can end). If +you want to forcefully delete them (after they are successfully flushed +to @code{stdout}) anyway, use @option{-delete} option. + +But you can verify produced stream after, by digesting it by yourself +with @option{-rx} and @option{-delete} options -- in that mode, stream +packets integrity will be checked and they will be deleted from the +spool if everything is good. So it is advisable to recheck your streams: + +@example +$ nncp-bundle -tx ALICE BOB WHATEVER | cdrecord -tao - +$ dd if=/dev/cd0 bs=2048 | nncp-bundle -rx -delete +@end example + +@option{-dryrun} option prevents any writes to the spool. This is +useful when you need to see what packets will pass by and possibly check +their integrity. diff --git a/doc/cmd/nncp-call.texi b/doc/cmd/nncp-call.texi new file mode 100644 index 0000000..9dc32bd --- /dev/null +++ b/doc/cmd/nncp-call.texi @@ -0,0 +1,60 @@ +@node nncp-call +@section nncp-call + +@example +$ nncp-call [options] + [-onlinedeadline INT] + [-maxonlinetime INT] + [-rx|-tx] + [-list] + [-pkts PKT,PKT,...] + [-rxrate INT] + [-txrate INT] + [-autotoss*] + [-nock] + NODE[:ADDR] [FORCEADDR] +@end example + +Call (connect to) specified @option{NODE} and run @ref{Sync, +synchronization} protocol with the @ref{nncp-daemon, daemon} on the +remote side. Normally this command could be run any time you wish to +either check for incoming packets, or to send out queued ones. +Synchronization protocol allows resuming and bidirectional packets +transfer. + +If @option{-rx} option is specified then only inbound packets +transmission is performed. If @option{-tx} option is specified, then +only outbound transmission is performed. + +@option{-onlinedeadline} overrides @ref{CfgOnlineDeadline, @emph{onlinedeadline}}. +@option{-maxonlinetime} overrides @ref{CfgMaxOnlineTime, @emph{maxonlinetime}}. +@option{-rxrate}/@option{-txrate} override @ref{CfgXxRate, rxrate/txrate}. + +@option{-list} option allows you to list packets of remote node, without +any transmission. You can specify what packets your want to download, by +specifying @option{-pkts} option with comma-separated list of packets +identifiers. + +Each @option{NODE} can contain several uniquely identified +@option{ADDR}esses in @ref{CfgAddrs, configuration} file. If you do +not specify the exact one, then all will be tried until the first +success. Optionally you can force @option{FORCEADDR} address usage, +instead of addresses taken from configuration file. You can specify both +@verb{|host:port|} and @verb{#|some command#} formats. + +@option{-autotoss} option runs tosser on node's spool every second +during the call. All @option{-autotoss-*} options is the same as in +@ref{nncp-toss} command. + +Partly downloaded packets are stored in @file{.part} files. By default +all downloaded files are sequentially checksummed in the background, +stripping @file{.part} extension if is successful. If @option{-nock} +option is set, then no checksumming is done, renaming fully downloaded +files to @file{.nock} extension. Pay attention that checksumming can be +time consuming and connection could be lost during that check, so remote +node won't be notified that the file is finished. If you run +@ref{nncp-check, @command{nncp-check -nock}}, that will checksum files +and strip the @file{.nock} extension, then repeated call to remote node +will notify about packet's completion. Also it will be notified if +@ref{nncp-toss, tossing} created @file{.seen} file. +Read @ref{CfgNoCK, more} about @option{-nock} option. diff --git a/doc/cmd/nncp-caller.texi b/doc/cmd/nncp-caller.texi new file mode 100644 index 0000000..0b309d8 --- /dev/null +++ b/doc/cmd/nncp-caller.texi @@ -0,0 +1,15 @@ +@node nncp-caller +@section nncp-caller + +@example +$ nncp-caller [options] [NODE ...] +@end example + +Croned daemon that calls remote nodes from time to time, according to +their @ref{CfgCalls, @emph{calls}} configuration field. + +Optional number of @option{NODE}s tells to ignore other ones. +Otherwise all nodes with specified @emph{calls} configuration +field will be called. + +Look at @ref{nncp-call} for more information. diff --git a/doc/cmd/nncp-cfgenc.texi b/doc/cmd/nncp-cfgenc.texi new file mode 100644 index 0000000..80fab22 --- /dev/null +++ b/doc/cmd/nncp-cfgenc.texi @@ -0,0 +1,39 @@ +@node nncp-cfgenc +@section nncp-cfgenc + +@example +$ nncp-cfgenc [options] [-s INT] [-t INT] [-p INT] cfg.hjson > cfg.hjson.eblob +$ nncp-cfgenc [options] -d cfg.hjson.eblob > cfg.hjson +@end example + +This command allows you to encrypt provided @file{cfg.hjson} file with +the passphrase, producing @ref{EBlob, eblob}, to safely keep your +configuration file with private keys. This utility was written for users +who do not want (or can not) to use either @url{https://gnupg.org/, +GnuPG} or similar tools. That @file{eblob} file can be used directly in +@option{-cfg} option of nearly all commands. + +@option{-s}, @option{-t}, @option{-p} are used to tune @file{eblob}'s +password strengthening function. Space memory cost (@option{-s}), +specified in number of BLAKE2b-256 blocks (32 bytes), tells how many +memory must be used for hashing -- bigger values are better, but slower. +Time cost (@option{-t}) tells how many rounds/iterations must be +performed -- bigger is better, but slower. Number of parallel jobs +(@option{-p}) tells how many computation processes will be run: this is +the same as running that number of independent hashers and then joining +their result together. + +When invoked for encryption, passphrase is entered manually twice. When +invoked for decryption (@option{-d} option), it is asked once and exits +if passphrase can not decrypt @file{eblob}. + +@option{-dump} options parses @file{eblob} and prints parameters used +during its creation. For example: +@example +$ nncp-cfgenc -dump /usr/local/etc/nncp.hjson.eblob +Strengthening function: Balloon with BLAKE2b-256 +Memory space cost: 1048576 bytes +Number of rounds: 16 +Number of parallel jobs: 2 +Blob size: 2494 +@end example diff --git a/doc/cmd/nncp-cfgmin.texi b/doc/cmd/nncp-cfgmin.texi new file mode 100644 index 0000000..d0125ac --- /dev/null +++ b/doc/cmd/nncp-cfgmin.texi @@ -0,0 +1,11 @@ +@node nncp-cfgmin +@section nncp-cfgmin + +@example +$ nncp-cfgmin [options] > stripped.hjson +@end example + +Print out stripped configuration version: only path to @ref{Spool, +spool}, path to log file, neighbours public keys are stayed. This is +useful mainly for usage with @ref{nncp-xfer} that has to know only +neighbours, without private keys involving. diff --git a/doc/cmd/nncp-cfgnew.texi b/doc/cmd/nncp-cfgnew.texi new file mode 100644 index 0000000..de3c71a --- /dev/null +++ b/doc/cmd/nncp-cfgnew.texi @@ -0,0 +1,17 @@ +@node nncp-cfgnew +@section nncp-cfgnew + +@example +$ nncp-cfgnew [options] [-area NAME] [-nocomments] > new.hjson +@end example + +Generate new node configuration: private keys, example configuration +file and print it to @code{stdout}. You must use this command when you +setup the new node. @option{-nocomments} will create configuration file +without descriptive huge comments -- useful for advanced users. + +With @option{-area} option you generate only the @ref{Area, area} +related part of the configuration file. + +Pay attention that private keys generation consumes an entropy from your +operating system. diff --git a/doc/cmd/nncp-check.texi b/doc/cmd/nncp-check.texi new file mode 100644 index 0000000..4a257af --- /dev/null +++ b/doc/cmd/nncp-check.texi @@ -0,0 +1,18 @@ +@node nncp-check +@section nncp-check + +@example +$ nncp-check [-nock] [-cycle INT] [options] +@end example + +Perform @ref{Spool, spool} directory integrity check. Read all files +that has Base32-encoded filenames and compare it with recalculated +@ref{MTH} hash output of their contents. + +The most useful mode of operation is with @option{-nock} option, that +checks integrity of @file{.nock} files, renaming them to ordinary +(verified) encrypted packets. + +@option{-cycle} option tells not to quit, but to repeat checking every +@option{INT} seconds in an infinite loop. That can be useful when +running this command as a daemon. diff --git a/doc/cmd/nncp-cronexpr.texi b/doc/cmd/nncp-cronexpr.texi new file mode 100644 index 0000000..f06202a --- /dev/null +++ b/doc/cmd/nncp-cronexpr.texi @@ -0,0 +1,24 @@ +@node nncp-cronexpr +@section nncp-cronexpr + +@example +$ nncp-cronexpr -num 12 "*/1 * * * * SAT,SUN 2021" +@end example + +Check validity of specified @ref{CronExpr, cron expression} and print 12 +next time entities: + +@example +$ nncp-cronexpr "*/5 * * * * * *" +Now: 2021-07-04T08:26:26.229285858Z +0: 2021-07-04T08:26:30Z +1: 2021-07-04T08:26:35Z +2: 2021-07-04T08:26:40Z +3: 2021-07-04T08:26:45Z +4: 2021-07-04T08:26:50Z +5: 2021-07-04T08:26:55Z +6: 2021-07-04T08:27:00Z +7: 2021-07-04T08:27:05Z +8: 2021-07-04T08:27:10Z +9: 2021-07-04T08:27:15Z +@end example diff --git a/doc/cmd/nncp-daemon.texi b/doc/cmd/nncp-daemon.texi new file mode 100644 index 0000000..aa11aca --- /dev/null +++ b/doc/cmd/nncp-daemon.texi @@ -0,0 +1,36 @@ +@node nncp-daemon +@section nncp-daemon + +@example +$ nncp-daemon [options] + [-maxconn INT] [-bind ADDR] [-inetd] + [-autotoss*] [-nock] [-mcd-once] +@end example + +Start listening TCP daemon, wait for incoming connections and run +@ref{Sync, synchronization protocol} with each of them. You can run +@ref{nncp-toss} utility in background to process inbound packets from +time to time. + +@option{-maxconn} option specifies how many simultaneous clients daemon +can handle. @option{-bind} option specifies @option{addr:port} it must +bind to and listen. + +It could be run as @command{inetd} service, by specifying +@option{-inetd} option. Pay attention that because it uses +@code{stdin}/@code{stdout}, it can not effectively work with IO timeouts +and connection closing can propagate up to 5 minutes in practice. +Example inetd-entry: + +@verbatim +uucp stream tcp6 nowait nncpuser /usr/local/bin/nncp-daemon nncp-daemon -quiet -inetd +@end verbatim + +@option{-autotoss} option runs tosser on node's spool every second +during the call. All @option{-autotoss-*} options is the same as in +@ref{nncp-toss} command. + +Read @ref{CfgNoCK, more} about @option{-nock} option. + +@option{-mcd-once} option sends @ref{MCD} announcements once and quits. +Could be useful with inetd-based setup, where daemons are not running. diff --git a/doc/cmd/nncp-exec.texi b/doc/cmd/nncp-exec.texi new file mode 100644 index 0000000..7bd7508 --- /dev/null +++ b/doc/cmd/nncp-exec.texi @@ -0,0 +1,51 @@ +@node nncp-exec +@section nncp-exec + +@example +$ nncp-exec [options] [-use-tmp] [-nocompress] NODE HANDLE [ARG0 ARG1 ...] +$ nncp-exec [options] [-use-tmp] [-nocompress] area:AREA HANDLE [ARG0 ARG1 ...] +@end example + +Send execution command to @option{NODE} for specified @option{HANDLE}. +Body is read from @code{stdin} into memory and compressed (unless +@option{-nocompress} is specified). After receiving, remote side will +execute specified @ref{CfgExec, handle} command with @option{ARG*} +appended and decompressed body fed to command's @code{stdin}. + +If @option{-use-tmp} option is specified, then @code{stdin} data is read +into temporary file first, requiring twice more disk space, but no +memory requirements. @ref{StdinTmpFile, Same temporary file} rules +applies as with @ref{nncp-file, nncp-file -} command. + +For example, if remote side has following configuration file for your +node: + +@verbatim +exec: { + sendmail: [/usr/sbin/sendmail, "-t"] + appender: ["/bin/sh", "-c", "cat >> /append"] +} +@end verbatim + +then executing @verb{|echo My message | nncp-exec -replynice 123 REMOTE +sendmail root@localhost|} will lead to execution of: + +@example +echo My message | + NNCP_SELF=REMOTE \ + NNCP_SENDER=OurNodeId \ + NNCP_NICE=123 \ + /usr/sbin/sendmail -t root@@localhost +@end example + +If @ref{CfgNotify, notification} is enabled on the remote side for exec +handles, then it will sent simple letter after successful command +execution with its output in message body. + +@strong{Pay attention} that packet generated with this command won't be +be chunked. + +If you use @option{area:AREA} instead of @option{NODE}, then +@ref{Multicast, multicast} packet will be sent to specified area. That +creates outgoing packet to the @strong{self} node, so you have to run +@ref{nncp-toss, tossing} to create outgoing packets to required subscribers. diff --git a/doc/cmd/nncp-file.texi b/doc/cmd/nncp-file.texi new file mode 100644 index 0000000..29f061a --- /dev/null +++ b/doc/cmd/nncp-file.texi @@ -0,0 +1,56 @@ +@node nncp-file +@section nncp-file + +@example +$ nncp-file [options] [-chunked INT] SRC NODE:[DST] +$ nncp-file [options] [-chunked INT] SRC area:AREA:[DST] +@end example + +Send @file{SRC} file to remote @option{NODE}. @file{DST} specifies +destination file name in remote's @ref{CfgIncoming, incoming} +directory. If this file already exists there, then counter will be +appended to it. + +This command queues file in @ref{Spool, spool} directory immediately +(through the temporary file of course) -- so pay attention that sending +2 GiB file will create 2 GiB outbound encrypted packet. + +@anchor{StdinTmpFile} +If @file{SRC} equals to @file{-}, then create an encrypted temporary +file and copy everything taken from @code{stdin} to it and use for outbound +packet creation. Pay attention that if you want to send 1 GiB of data +taken from @code{stdin}, then you have to have more than 2 GiB of disk space +for that temporary file and resulting encrypted packet. You can control +temporary file location directory with @env{TMPDIR} environment +variable. Encryption is performed in AEAD mode with +@url{https://cr.yp.to/chacha.html, ChaCha20}-@url{https://en.wikipedia.org/wiki/Poly1305, Poly1305} +algorithms. Data is divided on 128 KiB blocks. Each block is encrypted +with increasing nonce counter. File is deletes immediately after +creation, so even if program crashes -- disk space will be reclaimed, no +need in cleaning it up later. + +If @file{SRC} points to directory, then +@url{https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_01, pax archive} +will be created on the fly with directory contents and destination +filename @file{.tar} appended. It @strong{won't} contain any entities +metainformation, but modification time with the names. UID/GID are set +to zero. Directories have 777 permissions, files have 666, for being +friendly with @command{umask}. Also each entity will have comment like +@verb{|Autogenerated by NNCP version X.Y.Z built with goXXX|}. + +If @option{-chunked} is specified, then source file will be split +@ref{Chunked, on chunks}. @option{INT} is the desired chunk size in +KiBs. This mode is more CPU hungry. Pay attention that chunk is saved in +spool directory immediately and it is not deleted if any error occurs. +@option{-minsize} option is applied per each chunk. Do not forget about +@ref{ChunkedZFS, possible} ZFS deduplication issues. Zero +@option{-chunked} disables chunked transmission. + +If @ref{CfgNotify, notification} is enabled on the remote side for +file transmissions, then it will sent simple letter after successful +file receiving. + +If you use @option{area:AREA} instead of @option{NODE}, then +@ref{Multicast, multicast} packet will be sent to specified area. That +creates outgoing packet to the @strong{self} node, so you have to run +@ref{nncp-toss, tossing} to create outgoing packets to required subscribers. diff --git a/doc/cmd/nncp-freq.texi b/doc/cmd/nncp-freq.texi new file mode 100644 index 0000000..aa35f99 --- /dev/null +++ b/doc/cmd/nncp-freq.texi @@ -0,0 +1,15 @@ +@node nncp-freq +@section nncp-freq + +@example +$ nncp-freq [options] NODE:SRC [DST] +@end example + +Send file request to @option{NODE}, asking it to send its @file{SRC} +file from @ref{CfgFreq, freq.path} directory to our node under @file{DST} +filename in our @ref{CfgIncoming, incoming} one. If @file{DST} is not +specified, then last element of @file{SRC} will be used. + +If @ref{CfgNotify, notification} is enabled on the remote side for +file request, then it will sent simple letter after successful file +queuing. diff --git a/doc/cmd/nncp-hash.texi b/doc/cmd/nncp-hash.texi new file mode 100644 index 0000000..adc6e8f --- /dev/null +++ b/doc/cmd/nncp-hash.texi @@ -0,0 +1,17 @@ +@node nncp-hash +@section nncp-hash + +@example +$ nncp-hash [-file ...] [-seek X] [-debug] [-progress] +@end example + +Calculate @ref{MTH} hash of either stdin, or @option{-file} if +specified. + +You can optionally force seeking the file first, reading only part of +the file, and then prepending unread portion of data, with the +@option{-seek} option. It is intended only for testing and debugging of +MTH hasher capabilities. + +@option{-debug} option shows all intermediate MTH hashes. +And @option{-progress} will show progress bar. diff --git a/doc/cmd/nncp-log.texi b/doc/cmd/nncp-log.texi new file mode 100644 index 0000000..6ac6a7a --- /dev/null +++ b/doc/cmd/nncp-log.texi @@ -0,0 +1,9 @@ +@node nncp-log +@section nncp-log + +@example +$ nncp-log [options] +@end example + +Parse @ref{Log, log} file and print out its records in short +human-readable form. diff --git a/doc/cmd/nncp-pkt.texi b/doc/cmd/nncp-pkt.texi new file mode 100644 index 0000000..b3391b4 --- /dev/null +++ b/doc/cmd/nncp-pkt.texi @@ -0,0 +1,44 @@ +@node nncp-pkt +@section nncp-pkt + +@example +$ nncp-pkt [options] < pkt +$ nncp-pkt [options] [-decompress] -dump < pkt > payload +$ nncp-pkt -overheads +@end example + +Low level packet parser. Can be useful for debugging. There are two +types of packets: @ref{Plain, plain} and @ref{Encrypted, encrypted}. By +default it will print packet's header, for example: + +@example +Packet type: encrypted +Niceness: B (224) +Sender: 2WHBV3TPZHDOZGUJEH563ZEK7M33J4UESRFO4PDKWD5KZNPROABQ (self) +@end example + +@option{-dump} option outputs plain packet's payload (if it is file +transmission, then it will be the file itself as an example). If it is +an encrypted packet, then it will be decrypted first, outputing the +included plain packet, that can be fed to @command{nncp-pkt} again: + +@example +Packet type: plain +Payload type: transitional +Niceness: B (224) +Path: VHMTRWDOXPLK7BR55ICZ5N32ZJUMRKZEMFNGGCEAXV66GG43PEBQ (name-of-node) + +Packet type: plain +Payload type: exec compressed +Niceness: P (96) +Path: stargrave@@stargrave.org +@end example + +@option{-decompress} option tries to zstd-decompress data from plain +packet (useful with @verb{|exec compressed|} types of packets). + +@option{-overheads} options print encrypted, plain and size header overheads. + +This command automatically determines if an encrypted packet belongs to +@ref{Multicast, multicast} area and will try to decrypt it with its +corresponding key. diff --git a/doc/cmd/nncp-reass.texi b/doc/cmd/nncp-reass.texi new file mode 100644 index 0000000..7cae84b --- /dev/null +++ b/doc/cmd/nncp-reass.texi @@ -0,0 +1,58 @@ +@node nncp-reass +@section nncp-reass + +@example +$ nncp-reass [options] [-dryrun] [-keep] [-dump] [-stdout] FILE.nncp.meta +$ nncp-reass [options] [-dryrun] [-keep] @{-all | -node NODE@} +@end example + +Reassemble @ref{Chunked, chunked file} after @ref{nncp-toss, tossing}. + +When called with @option{FILE} option, this command will reassemble only +it. When called with @option{-node} option, this command will try to +reassemble all @file{.nncp.meta} files found in @option{NODE}'s +@ref{CfgIncoming, incoming} directory. When called with @option{-all} +option, then cycle through all known nodes to do the same. + +Reassembling process does the following: + +@enumerate +@item Parses @ref{Chunked, @file{.nncp.meta}} file. +@item Checks existence and size of every @file{.nncp.chunkXXX}. +@item Verifies integrity of every chunk. +@item Concatenates all chunks, simultaneously removing them from filesystem. +@end enumerate + +That process reads the whole data twice. Be sure to have free disk +space for at least one chunk. Decrypted chunk files as a rule are saved +in pseudo-random order, so removing them during reassembly process will +likely lead to filesystem fragmentation. Reassembly process on +filesystems with deduplication capability should be rather lightweight. + +If @option{-dryrun} option is specified, then only existence and +integrity checking are performed. + +If @option{-keep} option is specified, then no +@file{.nncp.meta}/@file{.nncp.chunkXXX} files are deleted during +reassembly process. + +@option{-stdout} option outputs reassembled file to @code{stdout}, +instead of saving to temporary file with renaming after. This could be +useful for reassembling on separate filesystem to lower fragmentation +effect, and/or separate storage device for higher performance. + +@option{-dump} option prints meta-file contents in human-friendly form. +It is useful mainly for debugging purposes. For example: +@example +Original filename: testfile +File size: 3.8 MiB (3987795 bytes) +Chunk size: 1.0 MiB (1048576 bytes) +Number of chunks: 4 +Checksums: + 0: eac60d819edf40b8ecdacd0b9a5a8c62de2d15eef3c8ca719eafa0be9b894017 + 1: 013a07e659f2e353d0e4339c3375c96c7fffaa2fa00875635f440bbc4631052a + 2: f4f883975a663f2252328707a30e71b2678f933b2f3103db8475b03293e4316e + 3: 0e9e229501bf0ca42d4aa07393d19406d40b179f3922a3986ef12b41019b45a3 +@end example + +Do not forget about @ref{ChunkedZFS, possible} ZFS deduplication issues. diff --git a/doc/cmd/nncp-rm.texi b/doc/cmd/nncp-rm.texi new file mode 100644 index 0000000..8d2db29 --- /dev/null +++ b/doc/cmd/nncp-rm.texi @@ -0,0 +1,53 @@ +@node nncp-rm +@section nncp-rm + +@example +$ nncp-rm [options] -tmp +$ nncp-rm [options] -lock +$ nncp-rm [options] @{-all|-node NODE@} -part +$ nncp-rm [options] @{-all|-node NODE@} -seen +$ nncp-rm [options] @{-all|-node NODE@} -nock +$ nncp-rm [options] @{-all|-node NODE@} -hdr +$ nncp-rm [options] @{-all|-node NODE@} -area +$ nncp-rm [options] @{-all|-node NODE@} [-rx] [-tx] +$ nncp-rm [options] @{-all|-node NODE@} -pkt PKT +@end example + +This command is aimed to delete various files from your spool directory: + +@itemize + +@item If @option{-tmp} option is specified, then it will delete all +temporary files in @file{spool/tmp} directory. Files may stay in it when +commands like @ref{nncp-file} fail for some reason. + +@item If @option{-lock} option is specified, then all @file{.lock} files +will be deleted in your spool directory. + +@item If @option{-pkt} option is specified, then @file{PKT} packet (its +Base32 name) will be deleted. This is useful when you see some packet +failing to be processed. + +@item When either @option{-rx} or @option{-tx} options are specified +(maybe both of them), then delete all packets from that given queues. + +@item @option{-part} option deletes @file{.part}ly downloaded files. + +@item @option{-seen} option deletes @file{.seen} files. But it does not +apply to @ref{Multicast, multicast areas} @file{.seen} ones! + +@item @option{-nock} option deletes non-checksummed (non-verified) +@file{.nock} files. + +@item @option{-hdr} option deletes cached @file{.hdr} files. + +@item @option{-area} option deletes @file{.seen} files in @file{area/} +subdirectories. + +@end itemize + +@option{-dryrun} option just prints what will be deleted. + +You can also select files that only have modification date older than specified +@option{-older} time units (@code{10s} (10 seconds), @code{5m} (5 minutes), +@code{12h} (12 hours), @code{2d} (2 days)). diff --git a/doc/cmd/nncp-stat.texi b/doc/cmd/nncp-stat.texi new file mode 100644 index 0000000..5759a01 --- /dev/null +++ b/doc/cmd/nncp-stat.texi @@ -0,0 +1,13 @@ +@node nncp-stat +@section nncp-stat + +@example +$ nncp-stat [options] [-pkt] [-node NODE] +@end example + +Print current @ref{Spool, spool} statistics about unsent and unprocessed +packets. For each node (unless @option{-node} specified) and each +niceness level there will be printed how many packets (with the total +size) are in inbound (Rx) and outbound (Tx) queues, how many +unchecksummed @file{.nock} packets or partly downloaded @file{.part} +ones. @option{-pkt} option show information about each packet. diff --git a/doc/cmd/nncp-toss.texi b/doc/cmd/nncp-toss.texi new file mode 100644 index 0000000..703c291 --- /dev/null +++ b/doc/cmd/nncp-toss.texi @@ -0,0 +1,32 @@ +@node nncp-toss +@section nncp-toss + +@example +$ nncp-toss [options] + [-node NODE] + [-dryrun] + [-cycle INT] + [-seen] + [-nofile] [-nofreq] [-noexec] [-notrns] [-noarea] +@end example + +Perform "tossing" operation on all inbound packets. This is the tool +that decrypts all packets and processes all payload packets in them: +copies files, sends mails, sends out file requests and relays transition +packets. It should be run after each online/offline exchange. + +@option{-dryrun} option does not perform any writing and sending, just +tells what it will do. + +@option{-cycle} option tells not to quit, but to repeat tossing every +@option{INT} seconds in an infinite loop. That can be useful when +running this command as a daemon. + +@option{-seen} option creates empty @file{XXX.seen} file after +successful tossing of @file{XXX} packet. @ref{nncp-xfer}, +@ref{nncp-bundle}, @ref{nncp-daemon} and @ref{nncp-call} commands skip +inbound packets that has been already seen, processed and tossed. This +is helpful to prevent duplicates. + +@option{-nofile}, @option{-nofreq}, @option{-noexec}, @option{-notrns}, +@option{-noarea} options allow to disable any kind of packet types processing. diff --git a/doc/cmd/nncp-xfer.texi b/doc/cmd/nncp-xfer.texi new file mode 100644 index 0000000..70edaf8 --- /dev/null +++ b/doc/cmd/nncp-xfer.texi @@ -0,0 +1,31 @@ +@node nncp-xfer +@section nncp-xfer + +@example +$ nncp-xfer [options] [-node NODE] [-mkdir] [-keep] [-rx|-tx] DIR +@end example + +Search for directory in @file{DIR} containing inbound packets for us and +move them to local @ref{Spool, spool} directory. Also search for known +neighbours directories and move locally queued outbound packets to them. +This command is used for offline packets transmission. + +If @option{-mkdir} option is specified, then outbound neighbour(s) +directories will be created. This is useful for the first time usage, +when storage device does not have any directories tree. + +If @option{-keep} option is specified, then keep copied files, do not +remove them. + +@option{-rx} option tells only to move inbound packets addressed to us. +@option{-tx} option tells exactly the opposite: move only outbound packets. + +@ref{nncp-cfgmin} could be useful for creating stripped minimalistic +configuration file version without any private keys. + +@file{DIR} directory has the following structure: +@file{RECIPIENT/SENDER/PACKET}, where @file{RECIPIENT} is Base32 encoded +destination node, @file{SENDER} is Base32 encoded sender node. + +Also look for @ref{nncp-bundle}, especially if you deal with CD-ROM and +tape drives. diff --git a/doc/cmds.texi b/doc/cmds.texi deleted file mode 100644 index 665b576..0000000 --- a/doc/cmds.texi +++ /dev/null @@ -1,656 +0,0 @@ -@node Commands -@unnumbered Commands - -Nearly all commands have the following common options: - -@table @option -@item -cfg - Path to configuration file. May be overridden by @env{NNCPCFG} - environment variable. If file file is an encrypted @ref{EBlob, - eblob}, then ask for passphrase to decrypt it first. -@item -debug - Print debug messages. Normally this option should not be used. -@item -minsize - @anchor{OptMinSize} - Minimal required resulting packet size, in KiBs. For example if you - send 2 KiB file and set @option{-minsize 4}, then resulting packet - will be 4 KiB (containing file itself and some junk). -@item -nice - Set desired outgoing packet @ref{Niceness, niceness level}. -@item -replynice - Set desired reply packet @ref{Niceness, niceness level}. Only freq - and exec packets look at that niceness level. -@item -via - Override @ref{CfgVia, via} configuration option for destination node. - Specified nodes must be separated with comma: @verb{|NODE1,NODE2|}. - With @verb{|-via -|} you can disable relaying at all. -@item -spool - Override path to spool directory. May be specified by - @env{NNCPSPOOL} environment variable. -@item -log - Override path to logfile. May be specified by @env{NNCPLOG} - environment variable. -@item -quiet - Print only errors, omit simple informational messages. In any case - those messages are logged, so you can reread them using - @ref{nncp-log} command. -@item -progress, -noprogress - Either force progress showing, or disable it. -@item -version - Print version information. -@item -warranty - Print warranty information (no warranty). -@end table - -@node nncp-bundle -@section nncp-bundle - -@example -$ nncp-bundle [options] -tx [-delete] NODE [NODE ...] > ... -$ nncp-bundle [options] -rx -delete [-dryrun] [NODE ...] < ... -$ nncp-bundle [options] -rx [-check] [-dryrun] [NODE ...] < ... -@end example - -With @option{-tx} option, this command creates @ref{Bundles, bundle} of -@ref{Encrypted, encrypted packets} from the spool directory and writes -it to @code{stdout}. - -With @option{-rx} option, this command takes bundle from @code{stdin} -and copies all found packets for our node to the spool directory. Pay -attention that @strong{no} integrity checking is done by default. Modern -tape drives could easily provide too much throughput your CPU won't be -able to verify on the fly. So if you won't @ref{nncp-toss, toss} -received packets at the place, it is advisable either to run -@ref{nncp-check} utility for packets integrity verification, or to use -@option{-check} option to enable on the fly integrity check. - -You can specify multiple @option{NODE} arguments, telling for what nodes -you want to create the stream, or take it from. If no nodes are -specified for @option{-rx} mode, then all packets aimed at us will be -processed. - -When packets are sent through the stream, they are still kept in the -spool directory, because there is no assurance that they are transferred -to the media (media (CD-ROM, tape drive, raw hard drive) can end). If -you want to forcefully delete them (after they are successfully flushed -to @code{stdout}) anyway, use @option{-delete} option. - -But you can verify produced stream after, by digesting it by yourself -with @option{-rx} and @option{-delete} options -- in that mode, stream -packets integrity will be checked and they will be deleted from the -spool if everything is good. So it is advisable to recheck your streams: - -@example -$ nncp-bundle -tx ALICE BOB WHATEVER | cdrecord -tao - -$ dd if=/dev/cd0 bs=2048 | nncp-bundle -rx -delete -@end example - -@option{-dryrun} option prevents any writes to the spool. This is -useful when you need to see what packets will pass by and possibly check -their integrity. - -@node nncp-call -@section nncp-call - -@example -$ nncp-call [options] - [-onlinedeadline INT] - [-maxonlinetime INT] - [-rx|-tx] - [-list] - [-pkts PKT,PKT,...] - [-rxrate INT] - [-txrate INT] - [-autotoss*] - [-nock] - NODE[:ADDR] [FORCEADDR] -@end example - -Call (connect to) specified @option{NODE} and run @ref{Sync, -synchronization} protocol with the @ref{nncp-daemon, daemon} on the -remote side. Normally this command could be run any time you wish to -either check for incoming packets, or to send out queued ones. -Synchronization protocol allows resuming and bidirectional packets -transfer. - -If @option{-rx} option is specified then only inbound packets -transmission is performed. If @option{-tx} option is specified, then -only outbound transmission is performed. - -@option{-onlinedeadline} overrides @ref{CfgOnlineDeadline, @emph{onlinedeadline}}. -@option{-maxonlinetime} overrides @ref{CfgMaxOnlineTime, @emph{maxonlinetime}}. -@option{-rxrate}/@option{-txrate} override @ref{CfgXxRate, rxrate/txrate}. -Read @ref{CfgNoCK, more} about @option{-nock} option. - -@option{-list} option allows you to list packets of remote node, without -any transmission. You can specify what packets your want to download, by -specifying @option{-pkts} option with comma-separated list of packets -identifiers. - -Each @option{NODE} can contain several uniquely identified -@option{ADDR}esses in @ref{CfgAddrs, configuration} file. If you do -not specify the exact one, then all will be tried until the first -success. Optionally you can force @option{FORCEADDR} address usage, -instead of addresses taken from configuration file. You can specify both -@verb{|host:port|} and @verb{#|some command#} formats. - -Pay attention that this command runs integrity check for each completely -received packet in the background. This can be time consuming. -Connection could be lost during that check and remote node won't be -notified that file is done. But after successful integrity check that -file is renamed from @file{.part} one and when you rerun -@command{nncp-call} again, remote node will receive completion -notification. - -@option{-autotoss} option runs tosser on node's spool every second -during the call. All @option{-autotoss-*} options is the same as in -@ref{nncp-toss} command. - -@node nncp-caller -@section nncp-caller - -@example -$ nncp-caller [options] [NODE ...] -@end example - -Croned daemon that calls remote nodes from time to time, according to -their @ref{CfgCalls, @emph{calls}} configuration field. - -Optional number of @option{NODE}s tells to ignore other ones. -Otherwise all nodes with specified @emph{calls} configuration -field will be called. - -Look at @ref{nncp-call} for more information. - -@node nncp-cfgenc -@section nncp-cfgenc - -@example -$ nncp-cfgenc [options] [-s INT] [-t INT] [-p INT] cfg.hjson > cfg.hjson.eblob -$ nncp-cfgenc [options] -d cfg.hjson.eblob > cfg.hjson -@end example - -This command allows you to encrypt provided @file{cfg.hjson} file with -the passphrase, producing @ref{EBlob, eblob}, to safely keep your -configuration file with private keys. This utility was written for users -who do not want (or can not) to use either @url{https://gnupg.org/, -GnuPG} or similar tools. That @file{eblob} file can be used directly in -@option{-cfg} option of nearly all commands. - -@option{-s}, @option{-t}, @option{-p} are used to tune @file{eblob}'s -password strengthening function. Space memory cost (@option{-s}), -specified in number of BLAKE2b-256 blocks (32 bytes), tells how many -memory must be used for hashing -- bigger values are better, but slower. -Time cost (@option{-t}) tells how many rounds/iterations must be -performed -- bigger is better, but slower. Number of parallel jobs -(@option{-p}) tells how many computation processes will be run: this is -the same as running that number of independent hashers and then joining -their result together. - -When invoked for encryption, passphrase is entered manually twice. When -invoked for decryption (@option{-d} option), it is asked once and exits -if passphrase can not decrypt @file{eblob}. - -@option{-dump} options parses @file{eblob} and prints parameters used -during its creation. For example: -@example -$ nncp-cfgenc -dump /usr/local/etc/nncp.hjson.eblob -Strengthening function: Balloon with BLAKE2b-256 -Memory space cost: 1048576 bytes -Number of rounds: 16 -Number of parallel jobs: 2 -Blob size: 2494 -@end example - -@node nncp-cfgmin -@section nncp-cfgmin - -@example -$ nncp-cfgmin [options] > stripped.hjson -@end example - -Print out stripped configuration version: only path to @ref{Spool, -spool}, path to log file, neighbours public keys are stayed. This is -useful mainly for usage with @ref{nncp-xfer} that has to know only -neighbours, without private keys involving. - -@node nncp-cfgnew -@section nncp-cfgnew - -@example -$ nncp-cfgnew [options] [-nocomments] > new.hjson -@end example - -Generate new node configuration: private keys, example configuration -file and print it to @code{stdout}. You must use this command when you -setup the new node. @option{-nocomments} will create configuration file -without descriptive huge comments -- useful for advanced users. - -Pay attention that private keys generation consumes an entropy from your -operating system. - -@node nncp-check -@section nncp-check - -@example -$ nncp-check [-nock] [options] -@end example - -Perform @ref{Spool, spool} directory integrity check. Read all files -that has Base32-encoded filenames and compare it with recalculated -@ref{MTH} hash output of their contents. - -The most useful mode of operation is with @option{-nock} option, that -checks integrity of @file{.nock} files, renaming them to ordinary -(verified) encrypted packets. - -@node nncp-cronexpr -@section nncp-cronexpr - -@example -$ nncp-cronexpr -num 12 "*/1 * * * * SAT,SUN 2021" -@end example - -Check validity of specified @ref{CronExpr, cron expression} and print 12 -next time entities. - -@node nncp-daemon -@section nncp-daemon - -@example -$ nncp-daemon [options] - [-maxconn INT] [-bind ADDR] [-inetd] - [-autotoss*] [-nock] [-mcd-once] -@end example - -Start listening TCP daemon, wait for incoming connections and run -@ref{Sync, synchronization protocol} with each of them. You can run -@ref{nncp-toss} utility in background to process inbound packets from -time to time. - -@option{-maxconn} option specifies how many simultaneous clients daemon -can handle. @option{-bind} option specifies @option{addr:port} it must -bind to and listen. - -It could be run as @command{inetd} service, by specifying -@option{-inetd} option. Pay attention that because it uses -@code{stdin}/@code{stdout}, it can not effectively work with IO timeouts -and connection closing can propagate up to 5 minutes in practice. -Example inetd-entry: - -@verbatim -uucp stream tcp6 nowait nncpuser /usr/local/bin/nncp-daemon nncp-daemon -quiet -inetd -@end verbatim - -@option{-autotoss} option runs tosser on node's spool every second -during the call. All @option{-autotoss-*} options is the same as in -@ref{nncp-toss} command. - -Read @ref{CfgNoCK, more} about @option{-nock} option. - -@option{-mcd-once} option sends @ref{MCD} announcements once and quits. -Could be useful with inetd-based setup, where daemons are not running. - -@node nncp-exec -@section nncp-exec - -@example -$ nncp-exec [options] [-use-tmp] [-nocompress] NODE HANDLE [ARG0 ARG1 ...] -@end example - -Send execution command to @option{NODE} for specified @option{HANDLE}. -Body is read from @code{stdin} into memory and compressed (unless -@option{-nocompress} is specified). After receiving, remote side will -execute specified @ref{CfgExec, handle} command with @option{ARG*} -appended and decompressed body fed to command's @code{stdin}. - -If @option{-use-tmp} option is specified, then @code{stdin} data is read -into temporary file first, requiring twice more disk space, but no -memory requirements. @ref{StdinTmpFile, Same temporary file} rules -applies as with @ref{nncp-file, nncp-file -} command. - -For example, if remote side has following configuration file for your -node: - -@verbatim -exec: { - sendmail: [/usr/sbin/sendmail, "-t"] - appender: ["/bin/sh", "-c", "cat >> /append"] -} -@end verbatim - -then executing @verb{|echo My message | nncp-exec -replynice 123 REMOTE -sendmail root@localhost|} will lead to execution of: - -@example -echo My message | - NNCP_SELF=REMOTE \ - NNCP_SENDER=OurNodeId \ - NNCP_NICE=123 \ - /usr/sbin/sendmail -t root@@localhost -@end example - -If @ref{CfgNotify, notification} is enabled on the remote side for exec -handles, then it will sent simple letter after successful command -execution with its output in message body. - -@strong{Pay attention} that packet generated with this command won't be -be chunked. - -@node nncp-file -@section nncp-file - -@example -$ nncp-file [options] [-chunked INT] SRC NODE:[DST] -@end example - -Send @file{SRC} file to remote @option{NODE}. @file{DST} specifies -destination file name in remote's @ref{CfgIncoming, incoming} -directory. If this file already exists there, then counter will be -appended to it. - -This command queues file in @ref{Spool, spool} directory immediately -(through the temporary file of course) -- so pay attention that sending -2 GiB file will create 2 GiB outbound encrypted packet. - -@anchor{StdinTmpFile} -If @file{SRC} equals to @file{-}, then create an encrypted temporary -file and copy everything taken from @code{stdin} to it and use for outbound -packet creation. Pay attention that if you want to send 1 GiB of data -taken from @code{stdin}, then you have to have more than 2 GiB of disk space -for that temporary file and resulting encrypted packet. You can control -temporary file location directory with @env{TMPDIR} environment -variable. Encryption is performed in AEAD mode with -@url{https://cr.yp.to/chacha.html, ChaCha20}-@url{https://en.wikipedia.org/wiki/Poly1305, Poly1305} -algorithms. Data is divided on 128 KiB blocks. Each block is encrypted -with increasing nonce counter. File is deletes immediately after -creation, so even if program crashes -- disk space will be reclaimed, no -need in cleaning it up later. - -If @file{SRC} points to directory, then -@url{https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_01, pax archive} -will be created on the fly with directory contents and destination -filename @file{.tar} appended. It @strong{won't} contain any entities -metainformation, but modification time with the names. UID/GID are set -to zero. Directories have 777 permissions, files have 666, for being -friendly with @command{umask}. Also each entity will have comment like -@verb{|Autogenerated by NNCP version X.Y.Z built with goXXX|}. - -If @option{-chunked} is specified, then source file will be split -@ref{Chunked, on chunks}. @option{INT} is the desired chunk size in -KiBs. This mode is more CPU hungry. Pay attention that chunk is saved in -spool directory immediately and it is not deleted if any error occurs. -@option{-minsize} option is applied per each chunk. Do not forget about -@ref{ChunkedZFS, possible} ZFS deduplication issues. Zero -@option{-chunked} disables chunked transmission. - -If @ref{CfgNotify, notification} is enabled on the remote side for -file transmissions, then it will sent simple letter after successful -file receiving. - -@node nncp-freq -@section nncp-freq - -@example -$ nncp-freq [options] NODE:SRC [DST] -@end example - -Send file request to @option{NODE}, asking it to send its @file{SRC} -file from @ref{CfgFreq, freq.path} directory to our node under @file{DST} -filename in our @ref{CfgIncoming, incoming} one. If @file{DST} is not -specified, then last element of @file{SRC} will be used. - -If @ref{CfgNotify, notification} is enabled on the remote side for -file request, then it will sent simple letter after successful file -queuing. - -@node nncp-hash -@section nncp-hash - -@example -$ nncp-log [-file ...] [-seek X] [-debug] [-progress] -@end example - -Calculate @ref{MTH} hash of either stdin, or @option{-file} if -specified. - -You can optionally force seeking the file first, reading only part of -the file, and then prepending unread portion of data, with the -@option{-seek} option. It is intended only for testing and debugging of -MTH hasher capabilities. - -@option{-debug} option shows all intermediate MTH hashes. -And @option{-progress} will show progress bar. - -@node nncp-log -@section nncp-log - -@example -$ nncp-log [options] -@end example - -Parse @ref{Log, log} file and print out its records in short -human-readable form. - -@node nncp-pkt -@section nncp-pkt - -@example -$ nncp-pkt [options] < pkt -$ nncp-pkt [options] [-decompress] -dump < pkt > payload -$ nncp-pkt -overheads -@end example - -Low level packet parser. Normally it should not be used, but can help in -debugging. - -By default it will print packet's type, for example: -@example -Packet type: encrypted -Niceness: 64 -Sender: 2WHBV3TPZHDOZGUJEH563ZEK7M33J4UESRFO4PDKWD5KZNPROABQ -@end example - -If you specify @option{-dump} option and provide an @ref{Encrypted, -encrypted} packet, then it will verify and decrypt it to @code{stdout}. -Encrypted packets contain @ref{Plain, plain} ones, that also can be fed -to @command{nncp-pkt}: - -@example -Packet type: plain -Payload type: transitional -Path: VHMTRWDOXPLK7BR55ICZ5N32ZJUMRKZEMFNGGCEAXV66GG43PEBQ - -Packet type: plain -Payload type: mail -Path: stargrave@@stargrave.org -@end example - -And with the @option{-dump} option it will give you the actual payload -(the whole file, mail message, and so on). @option{-decompress} option -tries to zstd-decompress the data from plain packet (useful for mail -packets). - -@option{-overheads} options print encrypted, plain and size header overheads. - -@node nncp-reass -@section nncp-reass - -@example -$ nncp-reass [options] [-dryrun] [-keep] [-dump] [-stdout] FILE.nncp.meta -$ nncp-reass [options] [-dryrun] [-keep] @{-all | -node NODE@} -@end example - -Reassemble @ref{Chunked, chunked file} after @ref{nncp-toss, tossing}. - -When called with @option{FILE} option, this command will reassemble only -it. When called with @option{-node} option, this command will try to -reassemble all @file{.nncp.meta} files found in @option{NODE}'s -@ref{CfgIncoming, incoming} directory. When called with @option{-all} -option, then cycle through all known nodes to do the same. - -Reassembling process does the following: - -@enumerate -@item Parses @ref{Chunked, @file{.nncp.meta}} file. -@item Checks existence and size of every @file{.nncp.chunkXXX}. -@item Verifies integrity of every chunk. -@item Concatenates all chunks, simultaneously removing them from filesystem. -@end enumerate - -That process reads the whole data twice. Be sure to have free disk -space for at least one chunk. Decrypted chunk files as a rule are saved -in pseudo-random order, so removing them during reassembly process will -likely lead to filesystem fragmentation. Reassembly process on -filesystems with deduplication capability should be rather lightweight. - -If @option{-dryrun} option is specified, then only existence and -integrity checking are performed. - -If @option{-keep} option is specified, then no -@file{.nncp.meta}/@file{.nncp.chunkXXX} files are deleted during -reassembly process. - -@option{-stdout} option outputs reassembled file to @code{stdout}, -instead of saving to temporary file with renaming after. This could be -useful for reassembling on separate filesystem to lower fragmentation -effect, and/or separate storage device for higher performance. - -@option{-dump} option prints meta-file contents in human-friendly form. -It is useful mainly for debugging purposes. For example: -@example -Original filename: testfile -File size: 3.8 MiB (3987795 bytes) -Chunk size: 1.0 MiB (1048576 bytes) -Number of chunks: 4 -Checksums: - 0: eac60d819edf40b8ecdacd0b9a5a8c62de2d15eef3c8ca719eafa0be9b894017 - 1: 013a07e659f2e353d0e4339c3375c96c7fffaa2fa00875635f440bbc4631052a - 2: f4f883975a663f2252328707a30e71b2678f933b2f3103db8475b03293e4316e - 3: 0e9e229501bf0ca42d4aa07393d19406d40b179f3922a3986ef12b41019b45a3 -@end example - - Do not forget about @ref{ChunkedZFS, possible} ZFS deduplication issues. - -@node nncp-rm -@section nncp-rm - -@example -$ nncp-rm [options] -tmp -$ nncp-rm [options] -lock -$ nncp-rm [options] -node NODE -part -$ nncp-rm [options] -node NODE -seen -$ nncp-rm [options] -node NODE -nock -$ nncp-rm [options] -node NODE [-rx] [-tx] -$ nncp-rm [options] -node NODE -pkt PKT -@end example - -This command is aimed to delete various files from your spool directory: - -@itemize - -@item If @option{-tmp} option is specified, then it will delete all -temporary files in @file{spool/tmp} directory. Files may stay in it when -commands like @ref{nncp-file} fail for some reason. - -@item If @option{-lock} option is specified, then all @file{.lock} files -will be deleted in your spool directory. - -@item If @option{-pkt} option is specified, then @file{PKT} packet (its -Base32 name) will be deleted. This is useful when you see some packet -failing to be processed. - -@item When either @option{-rx} or @option{-tx} options are specified -(maybe both of them), then delete all packets from that given queues. -@option{-part} option deletes @file{.part}ly downloaded files. -@option{-seen} option deletes @file{.seen} files. @option{-nock} option -deletes non-checksummed (non-verified) @file{.nock} files. - -@item @option{-dryrun} option just prints what will be deleted. - -@item You can also select files that only have modification date older -than specified @option{-older} time units (@code{10s} (10 seconds), -@code{5m} (5 minutes), @code{12h} (12 hours), @code{2d} (2 days)). - -@end itemize - -@node nncp-stat -@section nncp-stat - -@example -$ nncp-stat [options] [-pkt] [-node NODE] -@end example - -Print current @ref{Spool, spool} statistics about unsent and unprocessed -packets. For each node (unless @option{-node} specified) and each -niceness level there will be printed how many packets (with the total -size) are in inbound (Rx) and outbound (Tx) queues. @option{-pkt} option -show information about each packet. - -@node nncp-toss -@section nncp-toss - -@example -$ nncp-toss [options] - [-node NODE] - [-dryrun] - [-cycle INT] - [-seen] - [-nofile] - [-nofreq] - [-noexec] - [-notrns] -@end example - -Perform "tossing" operation on all inbound packets. This is the tool -that decrypts all packets and processes all payload packets in them: -copies files, sends mails, sends out file requests and relays transition -packets. It should be run after each online/offline exchange. - -@option{-dryrun} option does not perform any writing and sending, just -tells what it will do. - -@option{-cycle} option tells not to quit, but to repeat tossing every -@option{INT} seconds in an infinite loop. That can be useful when -running this command as a daemon. - -@option{-seen} option creates empty @file{XXX.seen} file after -successful tossing of @file{XXX} packet. @ref{nncp-xfer}, -@ref{nncp-bundle}, @ref{nncp-daemon} and @ref{nncp-call} commands skip -inbound packets that has been already seen, processed and tossed. This -is helpful to prevent duplicates. - -@option{-nofile}, @option{-nofreq}, @option{-noexec}, @option{-notrns} -options allow to disable any kind of packet types processing. - -@node nncp-xfer -@section nncp-xfer - -@example -$ nncp-xfer [options] [-node NODE] [-mkdir] [-keep] [-rx|-tx] DIR -@end example - -Search for directory in @file{DIR} containing inbound packets for us and -move them to local @ref{Spool, spool} directory. Also search for known -neighbours directories and move locally queued outbound packets to them. -This command is used for offline packets transmission. - -If @option{-mkdir} option is specified, then outbound neighbour(s) -directories will be created. This is useful for the first time usage, -when storage device does not have any directories tree. - -If @option{-keep} option is specified, then keep copied files, do not -remove them. - -@option{-rx} option tells only to move inbound packets addressed to us. -@option{-tx} option tells exactly the opposite: move only outbound packets. - -@ref{nncp-cfgmin} could be useful for creating stripped minimalistic -configuration file version without any private keys. - -@file{DIR} directory has the following structure: -@file{RECIPIENT/SENDER/PACKET}, where @file{RECIPIENT} is Base32 encoded -destination node, @file{SENDER} is Base32 encoded sender node. - -Also look for @ref{nncp-bundle}, especially if you deal with CD-ROM and -tape drives. diff --git a/doc/comparison.ru.texi b/doc/comparison.ru.texi index 280220c..933d157 100644 --- a/doc/comparison.ru.texi +++ b/doc/comparison.ru.texi @@ -14,6 +14,7 @@ @item Передача почты @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @item Передача новостей @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab Нет @item Передача файлов @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab Нет +@item Мультвещательная передача @tab Нет @tab @strong{Да} @tab @strong{Да} @tab Нет @item Разбиение файлов на части @tab Нет @tab @strong{Да} @tab @strong{Да} @tab Нет @item Удалённое исполнение команд @tab @strong{Да} @tab Нет @tab @strong{Да} @tab Нет @item Возобновляемое скачивание @tab @strong{Да} @tab @strong{Да} @tab @strong{Да} @tab Нет diff --git a/doc/comparison.texi b/doc/comparison.texi index 7bf4e53..d4e7ec6 100644 --- a/doc/comparison.texi +++ b/doc/comparison.texi @@ -13,6 +13,7 @@ FidoNet} Technology Networks) and @url{https://en.wikipedia.org/wiki/SMTP, SMTP} @item Mail transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @item News transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab No @item File transmission @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab No +@item Multicast transmission @tab No @tab @strong{Yes} @tab @strong{Yes} @tab No @item Chunked files @tab No @tab @strong{Yes} @tab @strong{Yes} @tab No @item Remote command execution @tab @strong{Yes} @tab No @tab @strong{Yes} @tab No @item Resumable downloads @tab @strong{Yes} @tab @strong{Yes} @tab @strong{Yes} @tab No diff --git a/doc/download.texi b/doc/download.texi index 4f920c1..f7707d9 100644 --- a/doc/download.texi +++ b/doc/download.texi @@ -28,171 +28,175 @@ Tarballs include all necessary required libraries: @multitable {XXXXX} {XXXX-XX-XX} {XXXX KiB} {link sign} {xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx} @headitem Version @tab Date @tab Size @tab Tarball @tab SHA256 checksum -@item @ref{Release 6.6.0, 6.6.0} @tab 2021-06-26 @tab 1041 KiB +@item @ref{Release 7_0_0, 7.0.0} @tab 2021-06-30 @tab 1123 KiB +@tab @url{download/nncp-7.0.0.tar.xz, link} @url{download/nncp-7.0.0.tar.xz.sig, sign} +@tab @code{D4D28E9A CF40FE12 68BDE134 9CD36076 282395BE 70094EFB 0DB75CE8 C32EA664} + +@item @ref{Release 6_6_0, 6.6.0} @tab 2021-06-26 @tab 1041 KiB @tab @url{download/nncp-6.6.0.tar.xz, link} @url{download/nncp-6.6.0.tar.xz.sig, sign} @tab @code{73DB666F A5C30282 770516B2 F39F1240 74117B45 A9F4B484 0361861A 183577F1} -@item @ref{Release 6.5.0, 6.5.0} @tab 2021-05-30 @tab 1041 KiB +@item @ref{Release 6_5_0, 6.5.0} @tab 2021-05-30 @tab 1041 KiB @tab @url{download/nncp-6.5.0.tar.xz, link} @url{download/nncp-6.5.0.tar.xz.sig, sign} @tab @code{241D2AA7 27275CCF 86F06797 1AA8B3B8 D625C85C 4279DFDE 560216E3 38670B9A} -@item @ref{Release 6.4.0, 6.4.0} @tab 2021-04-22 @tab 1042 KiB +@item @ref{Release 6_4_0, 6.4.0} @tab 2021-04-22 @tab 1042 KiB @tab @url{download/nncp-6.4.0.tar.xz, link} @url{download/nncp-6.4.0.tar.xz.sig, sign} @tab @code{3D0D1156 D69AF698 D402663C F84E51CC 3D40A50D 300E34D1 105A6F75 32E4B99B} -@item @ref{Release 6.3.0, 6.3.0} @tab 2021-04-14 @tab 1042 KiB +@item @ref{Release 6_3_0, 6.3.0} @tab 2021-04-14 @tab 1042 KiB @tab @url{download/nncp-6.3.0.tar.xz, link} @url{download/nncp-6.3.0.tar.xz.sig, sign} @tab @code{76C26A11 E3423540 BB7B8470 820176A3 5FCD0493 B21A872E C223EB94 43BA466B} -@item @ref{Release 6.2.1, 6.2.1} @tab 2021-03-26 @tab 1038 KiB +@item @ref{Release 6_2_1, 6.2.1} @tab 2021-03-26 @tab 1038 KiB @tab @url{download/nncp-6.2.1.tar.xz, link} @url{download/nncp-6.2.1.tar.xz.sig, sign} @tab @code{D9682D95 4D68025A F5B07516 258D9FFC DA29A4D7 E7E1635B E0C219A1 C5DDB067} -@item @ref{Release 6.2.0, 6.2.0} @tab 2021-03-07 @tab 1038 KiB +@item @ref{Release 6_2_0, 6.2.0} @tab 2021-03-07 @tab 1038 KiB @tab @url{download/nncp-6.2.0.tar.xz, link} @url{download/nncp-6.2.0.tar.xz.sig, sign} @tab @code{272CEDED 69FFF3B3 78767297 3199481A C610B753 BB82C22E ECEC45FC 05DA40FE} -@item @ref{Release 6.1.0, 6.1.0} @tab 2021-02-24 @tab 1040 KiB +@item @ref{Release 6_1_0, 6.1.0} @tab 2021-02-24 @tab 1040 KiB @tab @url{download/nncp-6.1.0.tar.xz, link} @url{download/nncp-6.1.0.tar.xz.sig, sign} @tab @code{083A533F 7D021206 9AE07F9F D6CD22E3 C5BE09E8 30F2C9C4 97D97CF6 14E5413F} -@item @ref{Release 6.0.0, 6.0.0} @tab 2021-01-23 @tab 1028 KiB +@item @ref{Release 6_0_0, 6.0.0} @tab 2021-01-23 @tab 1028 KiB @tab @url{download/nncp-6.0.0.tar.xz, link} @url{download/nncp-6.0.0.tar.xz.sig, sign} @tab @code{42FE8AA5 4520B3A1 ABB50D66 1BBBA6A1 41CE4E74 9B4816B0 D4C6845D 67465916} -@item @ref{Release 5.6.0, 5.6.0} @tab 2021-01-17 @tab 1024 KiB +@item @ref{Release 5_6_0, 5.6.0} @tab 2021-01-17 @tab 1024 KiB @tab @url{download/nncp-5.6.0.tar.xz, link} @url{download/nncp-5.6.0.tar.xz.sig, sign} @tab @code{1DC83F05 F14A3C3B 95820046 C60B170E B8C8936F 142A5B9A 1E943E6F 4CEFBDE3} -@item @ref{Release 5.5.1, 5.5.1} @tab 2021-01-11 @tab 1165 KiB +@item @ref{Release 5_5_1, 5.5.1} @tab 2021-01-11 @tab 1165 KiB @tab @url{download/nncp-5.5.1.tar.xz, link} @url{download/nncp-5.5.1.tar.xz.sig, sign} @tab @code{E7DEED7A D3BA696C F64359C0 DC0A93AD 109950C5 6660D028 5FD7BB57 120C9CF7} -@item @ref{Release 5.5.0, 5.5.0} @tab 2021-01-07 @tab 1161 KiB +@item @ref{Release 5_5_0, 5.5.0} @tab 2021-01-07 @tab 1161 KiB @tab @url{download/nncp-5.5.0.tar.xz, link} @url{download/nncp-5.5.0.tar.xz.sig, sign} @tab @code{EF0CBEE1 520BE97D A210794C 172BF444 E6F75DB2 84F5BD05 66919193 326AED77} -@item @ref{Release 5.4.1, 5.4.1} @tab 2020-09-28 @tab 1143 KiB +@item @ref{Release 5_4_1, 5.4.1} @tab 2020-09-28 @tab 1143 KiB @tab @url{download/nncp-5.4.1.tar.xz, link} @url{download/nncp-5.4.1.tar.xz.sig, sign} @tab @code{A02D0C9B 51533DF8 115C17E1 02F8C485 9F7B805A 64290CDF 79151BA9 E627FA63} -@item @ref{Release 5.3.3, 5.3.3} @tab 2020-01-23 @tab 1116 KiB +@item @ref{Release 5_3_3, 5.3.3} @tab 2020-01-23 @tab 1116 KiB @tab @url{download/nncp-5.3.3.tar.xz, link} @url{download/nncp-5.3.3.tar.xz.sig, sign} @tab @code{707CD852 4E424C24 BCB22D6B 4BC81709 71C42A5F E0062B93 A8D1DD9D 7FB365D0} -@item @ref{Release 5.3.2, 5.3.2} @tab 2019-12-28 @tab 1118 KiB +@item @ref{Release 5_3_2, 5.3.2} @tab 2019-12-28 @tab 1118 KiB @tab @url{download/nncp-5.3.2.tar.xz, link} @url{download/nncp-5.3.2.tar.xz.sig, sign} @tab @code{6E2D1B3C CA0DD462 A6F5F8DE 5CB8DE15 C3D33C74 238A2C52 373C7BD6 A126A834} -@item @ref{Release 5.3.1, 5.3.1} @tab 2019-12-25 @tab 1117 KiB +@item @ref{Release 5_3_1, 5.3.1} @tab 2019-12-25 @tab 1117 KiB @tab @url{download/nncp-5.3.1.tar.xz, link} @url{download/nncp-5.3.1.tar.xz.sig, sign} @tab @code{23A52819 F0395A6A E05E4176 017DCA3C 4A20A023 EEADA6A3 3168E58D BEE34A5B} -@item @ref{Release 5.3.0, 5.3.0} @tab 2019-12-22 @tab 1112 KiB +@item @ref{Release 5_3_0, 5.3.0} @tab 2019-12-22 @tab 1112 KiB @tab @url{download/nncp-5.3.0.tar.xz, link} @url{download/nncp-5.3.0.tar.xz.sig, sign} @tab @code{9F093115 506D00E7 2E41ACD6 3F283172 8430E1C2 8BA4A941 FFA3C65D 89AD4ED0} -@item @ref{Release 5.2.1, 5.2.1} @tab 2019-12-15 @tab 1109 KiB +@item @ref{Release 5_2_1, 5.2.1} @tab 2019-12-15 @tab 1109 KiB @tab @url{download/nncp-5.2.1.tar.xz, link} @url{download/nncp-5.2.1.tar.xz.sig, sign} @tab @code{983D1A8A 4398C281 76356AE1 C5541124 B0755555 D115063B D1388F85 9C4A6B3E} -@item @ref{Release 5.2.0, 5.2.0} @tab 2019-12-14 @tab 1109 KiB +@item @ref{Release 5_2_0, 5.2.0} @tab 2019-12-14 @tab 1109 KiB @tab @url{download/nncp-5.2.0.tar.xz, link} @url{download/nncp-5.2.0.tar.xz.sig, sign} @tab @code{FFC55467 8B4ECCA6 92D90F42 ACC0286D 209E054E EA1CBF87 0307003E CF219610} -@item @ref{Release 5.1.2, 5.1.2} @tab 2019-12-13 @tab 1106 KiB +@item @ref{Release 5_1_2, 5.1.2} @tab 2019-12-13 @tab 1106 KiB @tab @url{download/nncp-5.1.2.tar.xz, link} @url{download/nncp-5.1.2.tar.xz.sig, sign} @tab @code{52B2043B 1B22D20F C44698EC AFE5FF46 F99B4DD5 2C392D4D 25FE1580 993263B3} -@item @ref{Release 5.1.1, 5.1.1} @tab 2019-12-01 @tab 1103 KiB +@item @ref{Release 5_1_1, 5.1.1} @tab 2019-12-01 @tab 1103 KiB @tab @url{download/nncp-5.1.1.tar.xz, link} @url{download/nncp-5.1.1.tar.xz.sig, sign} @tab @code{B9537678 E5B549BA 6FA0D20D 41B2D4A9 4ED31F2C AB9FAF63 A388D95E 7662A93F} -@item @ref{Release 5.1.0, 5.1.0} @tab 2019-11-24 @tab 1103 KiB +@item @ref{Release 5_1_0, 5.1.0} @tab 2019-11-24 @tab 1103 KiB @tab @url{download/nncp-5.1.0.tar.xz, link} @url{download/nncp-5.1.0.tar.xz.sig, sign} @tab @code{6F5B74EC 952EAFEC 2A787463 CE1E808E CC990F03 D46F28E9 A89BAB55 5A2C2214} -@item @ref{Release 5.0.0, 5.0.0} @tab 2019-11-15 @tab 1099 KiB +@item @ref{Release 5_0_0, 5.0.0} @tab 2019-11-15 @tab 1099 KiB @tab @url{download/nncp-5.0.0.tar.xz, link} @url{download/nncp-5.0.0.tar.xz.sig, sign} @tab @code{3696D7EE B0783E91 87E5EEF4 EFC35235 10452353 7C51FA4C 9BD3CBEE A22678B3} -@item @ref{Release 4.1, 4.1} @tab 2019-05-01 @tab 1227 KiB +@item @ref{Release 4_1, 4.1} @tab 2019-05-01 @tab 1227 KiB @tab @url{download/nncp-4.1.tar.xz, link} @url{download/nncp-4.1.tar.xz.sig, sign} @tab @code{29AEC53D EC914906 D7C47194 0955A32E 2BF470E6 9B8E09D3 AF3B62D8 CC8E541E} -@item @ref{Release 4.0, 4.0} @tab 2019-04-28 @tab 1227 KiB +@item @ref{Release 4_0, 4.0} @tab 2019-04-28 @tab 1227 KiB @tab @url{download/nncp-4.0.tar.xz, link} @url{download/nncp-4.0.tar.xz.sig, sign} @tab @code{EAFA6272 22E355FC EB772A90 FC6DEA8E AE1F1695 3F48A4A3 57ADA0B4 FF918452} -@item @ref{Release 3.4, 3.4} @tab 2018-06-10 @tab 1154 KiB +@item @ref{Release 3_4, 3.4} @tab 2018-06-10 @tab 1154 KiB @tab @url{download/nncp-3.4.tar.xz, link} @url{download/nncp-3.4.tar.xz.sig, sign} @tab @code{9796C4CB 7B670FC7 5FEED3CD 467CA556 B230387D 935B09BB 4B19FD57 FD17FFBA} -@item @ref{Release 3.3, 3.3} @tab 2018-06-02 @tab 1152 KiB +@item @ref{Release 3_3, 3.3} @tab 2018-06-02 @tab 1152 KiB @tab @url{download/nncp-3.3.tar.xz, link} @url{download/nncp-3.3.tar.xz.sig, sign} @tab @code{1F8FA9B4 6125D8A9 0608298B A1ED87E1 12DB2D8B 81C766DE F4DFE191 C7B1BFC2} -@item @ref{Release 3.2, 3.2} @tab 2018-05-27 @tab 1147 KiB +@item @ref{Release 3_2, 3.2} @tab 2018-05-27 @tab 1147 KiB @tab @url{download/nncp-3.2.tar.xz, link} @url{download/nncp-3.2.tar.xz.sig, sign} @tab @code{BE76802F 1E273D1D E91F0648 A7CB23C5 989F5390 A36F2D0C FD873046 51B9141E} -@item @ref{Release 3.1, 3.1} @tab 2018-02-18 @tab 1145 KiB +@item @ref{Release 3_1, 3.1} @tab 2018-02-18 @tab 1145 KiB @tab @url{download/nncp-3.1.tar.xz, link} @url{download/nncp-3.1.tar.xz.sig, sign} @tab @code{B9344516 4230B58E 8AAADAA2 066F37F2 493CCB71 B025126B BCAD8FAD 6535149F} -@item @ref{Release 3.0, 3.0} @tab 2017-12-30 @tab 993 KiB +@item @ref{Release 3_0, 3.0} @tab 2017-12-30 @tab 993 KiB @tab @url{download/nncp-3.0.tar.xz, link} @url{download/nncp-3.0.tar.xz.sig, sign} @tab @code{248B2257 2F576E79 A19672E9 B82EB649 18FC95A9 194408C0 67EA4DD3 0468286D} -@item @ref{Release 2.0, 2.0} @tab 2017-12-02 @tab 986 KiB +@item @ref{Release 2_0, 2.0} @tab 2017-12-02 @tab 986 KiB @tab @url{download/nncp-2.0.tar.xz, link} @url{download/nncp-2.0.tar.xz.sig, sign} @tab @code{BEF31B13 FB25381E A511FB77 067798AB 27409238 BDF5600F E2EADB29 E5E78996} -@item @ref{Release 1.0, 1.0} @tab 2017-12-02 @tab 987 KiB +@item @ref{Release 1_0, 1.0} @tab 2017-12-02 @tab 987 KiB @tab @url{download/nncp-1.0.tar.xz, link} @url{download/nncp-1.0.tar.xz.sig, sign} @tab @code{68BF7803 CD25F59A 56D9FD6C 695002B5 BFBAF591 8A6583F4 3139FC28 CA1AB4AF} -@item @ref{Release 0.12, 0.12} @tab 2017-10-08 @tab 978 KiB +@item @ref{Release 0_12, 0.12} @tab 2017-10-08 @tab 978 KiB @tab @url{download/nncp-0.12.tar.xz, link} @url{download/nncp-0.12.tar.xz.sig, sign} @tab @code{707B4005 97753B29 73A5F3E5 DAB51B92 21CC296D 690EF4BC ADE93E0D 2595A5F2} -@item @ref{Release 0.11, 0.11} @tab 2017-08-21 @tab 1031 KiB +@item @ref{Release 0_11, 0.11} @tab 2017-08-21 @tab 1031 KiB @tab @url{download/nncp-0.11.tar.xz, link} @url{download/nncp-0.11.tar.xz.sig, sign} @tab @code{D0F73C3B ADBF6B8B 13641A61 4D34F65F 20AF4C84 90894331 BF1F1609 2D65E719} -@item @ref{Release 0.10, 0.10} @tab 2017-07-04 @tab 949 KiB +@item @ref{Release 0_10, 0.10} @tab 2017-07-04 @tab 949 KiB @tab @url{download/nncp-0.10.tar.xz, link} @url{download/nncp-0.10.tar.xz.sig, sign} @tab @code{DCE7C762 2F9281EB 282F1A67 5CA6500E 854F2DEC D60F3264 07872B91 4F4E6FA0} -@item @ref{Release 0.9, 0.9} @tab 2017-05-17 @tab 942 KiB +@item @ref{Release 0_9, 0.9} @tab 2017-05-17 @tab 942 KiB @tab @url{download/nncp-0.9.tar.xz, link} @url{download/nncp-0.9.tar.xz.sig, sign} @tab @code{8D0765A5 F9D81086 7E1F5AB4 52A9464D C5035CCB 4E09A29A 9C9A4934 1A72AB2C} -@item @ref{Release 0.8, 0.8} @tab 2017-04-30 @tab 932 KiB +@item @ref{Release 0_8, 0.8} @tab 2017-04-30 @tab 932 KiB @tab @url{download/nncp-0.8.tar.xz, link} @url{download/nncp-0.8.tar.xz.sig, sign} @tab @code{9BD607D5 C5551857 B7E9277D 0E857936 1DB7353A E0F1556E EA9B1D91 8305B184} -@item @ref{Release 0.7, 0.7} @tab 2017-04-02 @tab 783 KiB +@item @ref{Release 0_7, 0.7} @tab 2017-04-02 @tab 783 KiB @tab @url{download/nncp-0.7.tar.xz, link} @url{download/nncp-0.7.tar.xz.sig, sign} @tab @code{D3407323 F89296DD 743FA764 51964B43 794E61BE 0E1D2DD4 ABD02042 B94FFC4F} -@item @ref{Release 0.6, 0.6} @tab 2017-02-05 @tab 746 KiB +@item @ref{Release 0_6, 0.6} @tab 2017-02-05 @tab 746 KiB @tab @url{download/nncp-0.6.tar.xz, link} @url{download/nncp-0.6.tar.xz.sig, sign} @tab @code{DCFEE3F9 F669AC28 563C50DB 67BB8B43 0CFF4AB6 EC770ACE B5378D0B B40C0656} -@item @ref{Release 0.5, 0.5} @tab 2017-01-19 @tab 743 KiB +@item @ref{Release 0_5, 0.5} @tab 2017-01-19 @tab 743 KiB @tab @url{download/nncp-0.5.tar.xz, link} @url{download/nncp-0.5.tar.xz.sig, sign} @tab @code{D98F9149 5A6D6726 4C659640 1AD7F400 271A58CE 5D8D4AC5 5D1CF934 59BEDFA6} -@item @ref{Release 0.4, 0.4} @tab 2017-01-17 @tab 741 KiB +@item @ref{Release 0_4, 0.4} @tab 2017-01-17 @tab 741 KiB @tab @url{download/nncp-0.4.tar.xz, link} @url{download/nncp-0.4.tar.xz.sig, sign} @tab @code{93577327 B3DEBFE3 A80BEB0D 8325B2E6 0939EC55 4DBB05F3 4CA34B99 229C3722} -@item @ref{Release 0.3, 0.3} @tab 2017-01-17 @tab 741 KiB +@item @ref{Release 0_3, 0.3} @tab 2017-01-17 @tab 741 KiB @tab @url{download/nncp-0.3.tar.xz, link} @url{download/nncp-0.3.tar.xz.sig, sign} @tab @code{6E76EC5E 6B575C65 BF2D6388 870F2A1C 417D63E4 1628CAA1 BB499D0D 0634473B} -@item @ref{Release 0.2, 0.2} @tab 2017-01-17 @tab 740 KiB +@item @ref{Release 0_2, 0.2} @tab 2017-01-17 @tab 740 KiB @tab @url{download/nncp-0.2.tar.xz, link} @url{download/nncp-0.2.tar.xz.sig, sign} @tab @code{00BEAC5A 0C4083B0 42E3152B ACA6FF20 12768B82 CE24D716 8E04279C ECE14DB7} diff --git a/doc/index.texi b/doc/index.texi index 27be3bb..000dab9 100644 --- a/doc/index.texi +++ b/doc/index.texi @@ -43,6 +43,7 @@ There are also articles about its usage outside this website: * Installation:: * Configuration:: * Call configuration: Call. +* Multicast areas: Multicast. * Integration:: * Commands:: * Administration:: @@ -67,17 +68,18 @@ There are also articles about its usage outside this website: @include news.texi @include russian.texi @include install.texi -@include cfg.texi +@include cfg/index.texi @include call.texi +@include multicast.texi @include integration.texi -@include cmds.texi +@include cmd/index.texi @include admin.texi @include niceness.texi @include chunked.texi @include bundles.texi @include spool.texi @include log.texi -@include pkt.texi +@include pkt/index.texi @include mth.texi @include sp.texi @include mcd.texi diff --git a/doc/install.texi b/doc/install.texi index 3ca13b1..a8189f1 100644 --- a/doc/install.texi +++ b/doc/install.texi @@ -14,7 +14,7 @@ Possibly NNCP package already exists for your distribution: NNCP should run on any POSIX-compatible operating system. NNCP is written on @url{https://golang.org/, Go} programming language -and you have to install Go compiler 1.12+ version. @command{Make} (BSD +and you have to install Go compiler 1.13+ version. @command{Make} (BSD and GNU versions are fine) is recommended for convenient building. @url{https://www.gnu.org/software/texinfo/, Texinfo} is used for building documentation (although tarballs already include it). diff --git a/doc/multicast.texi b/doc/multicast.texi new file mode 100644 index 0000000..c330426 --- /dev/null +++ b/doc/multicast.texi @@ -0,0 +1,148 @@ +@node Multicast +@unnumbered Multicast areas + +NNCP has ability to multicast packets: send single packet to multiple +recipients, which also can send it further to others. It can also be +called echomail (like in FidoNet networks) or newsgroup (like in Usenet +networks). + +@anchor{Area} +Each multicast group is identified by so-called @strong{area}. Area +consists of private/public Curve25519 keypairs for @ref{Encrypted area, +packets encryption}, identity (BLAKE2b-256 hash of the public key) and +possible subscribers. + +You can make either file or exec transmissions to the areas. Those +ordinary file/exec packets are double wrapped in: + +@itemize +@item encrypted packet, securing the actual packet contents from +participants not having area's keypairs (but still being able to relay +that encrypted packet to the others) +@item area packet, containing area's identity, telling that tossing node +can should it to the subscribers further +@end itemize + +Area's message identity (@code{MsgHash}) is the hash of the encrypted +packet header. Because the area packet, containing the encrypted packet, +is relayed as-is without any modifications, that area message's hash +will be the same on each node it reaches. + +@ref{nncp-toss, Tosser}'s algorithm of processing the area packet is +following: + +@itemize +@item check is it known area's identity (@code{AREA}). + Fail/skip if it is unknown +@item hash encrypted packet's header, getting the @code{MsgHash} +@item for each area's subscribers: + @itemize + @item check if that message was already seen (sent or received from) + before by the destination node: check existence of + @file{SPOOL/NODE/area/AREA/MsgHash.seen} file. Skip that node if + it exists + @item if subscriber's node is not the one we received the packet + from, then create outgoing encrypted packet to it, with that + area packet inside + @item create corresponding @file{MsgHash.seen} file + @item "rewind" the outer encrypted file to the beginning and repeat + the whole cycle again, while all of subscribers will "seen" that + area's message. + + Expensive signature verification and shared key computation + procedures are skipped in the following cycles -- only symmetric + cryptography will be in use, having negligible CPU resource + consumption. + @end itemize +@item check if we have seen that area's message before by looking at + @file{SPOOL/SELF/area/AREA/MsgHash.seen}. If so, remove the packet, + because it is just a ordinary possible duplicate, finish its processing +@item check if we have got corresponding area's private key. If no key + exists, then remove the packet, finish its processing -- we just + relay it further without being able to read it +@item look if area's encrypted packet's sender is known to us. If + neither it is known, nor we have @code{allow-unknown} configuration + option set for that area, then fail +@item otherwise start decryption procedure, possibly ignoring the + sender's signature verification if it is unknown +@item fed the decrypted contents to the toss-procedure as an ordinary + plain packet, receiving files or exec calls +@item mark area's message as the seen one, remove the packet, finish + processing +@end itemize + +Because outgoing packets creation for each subscriber can be time and +(disk) resource consuming, we can suddenly fail. It would be bad if we +will loose the possibility to retry the multicasting process again. So +we have got to save somehow outgoing area's message in permanent +storage, while outgoing copies are created. That is why the initial (not +relaying) message to the area is sent to the @strong{self} and processed +by the @ref{nncp-toss, tosser} to create necessary outgoing message +copies. Because message to myself is also encrypted, area's message is +encrypted and secured and noone sees plaintext @code{MsgHash}, knowing +that you either originated or have that message on the disk. + +For example we have got 4 nodes participating in the single area and +let's send file to that area from the @code{nodeA}: + +@example +nodeA -> subs: ["nodeB", "nodeD"] +nodeB -> subs: ["nodeC", "nodeD", "nodeA"], no keys +nodeC -> subs: ["nodeB"] +nodeD -> subs: ["nodeA", "nodeB"] +@end example + +@example +A -- B -- C +\ / + \ / + D +@end example + +@example +$ nncp-file nodelist-20210704.rec area:nodelist-updates: +$ nncp-toss -node self +@end example + +@enumerate +@item +@command{nncp-file} creates an encrypted packet with area packet and +encrypted packet inside it, with our own @code{self} node as a recipient +(in the @file{SPOOL/SELF/tx} directory). It also creates the +@file{SPOOL/SELF/area/AREA/MSGHASH.seen} file. + +@item +@command{nncp-toss} sees @file{tx/} file and "opens" it, applying the +area message tossing procedure as described above. That will create +outgoing packets in @file{SPOOL/nodeB/tx} and @file{SPOOL/nodeD/tx} +directories with @file{SPOOL/nodeB/area/AREA/MSGHASH.seen} +@file{SPOOL/nodeD/area/AREA/MSGHASH.seen} files. Because we already have +@file{SPOOL/SELF/area/AREA/MSGHASH.seen}, that packet is removed then. + +@item +When @code{nodeB} receives the encrypted packet, it sees the area one +inside. It copies/relays it to the @code{nodeC} and @code{nodeD}. It can +not read area's message because it lacks the keys. + +@item +@code{nodeC} does not relay it to anyone. Just stores +@file{nodelist-20210704.rec} in the incoming directory. + +@item +@code{nodeD} receives packets from both @code{nodeA} and @code{nodeB}. +Only one of them processed, and other is ignored because corresponding +@file{MSGHASH.seen} file will exist. + +If @code{nodeD} will receive packet from the @code{nodeB} first, it will +relay it to the @code{nodeA} also, that will silently remove it when +tossing, because it was already seen. + +@strong{TODO}: we must not relay packet to the node also presenting as +the sender of the area's message. Obviously it has seen it. + +@item +When @code{nodeC} sends message to the area, then @code{nodeA} will +receive it twice from @code{nodeB} and @code{nodeD}, ignoring one of +them during tossing. + +@end enumerate diff --git a/doc/news.ru.texi b/doc/news.ru.texi index c045d75..49cecae 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -1,10 +1,49 @@ @node Новости @section Новости +@node Релиз 7.1.0 +@subsection Релиз 7.1.0 +@itemize + +@item +Исправлена работоспособность @command{nncp-file} и @command{nncp-exec} +команд использующих временный файл (stdin и @option{-use-tmp}). + +@item +Исправлен пропадающий плохой код возврата в @command{nncp-exec} команде. + +@item +Исправлено некорректное генерирование @file{.hdr} при использовании +транзитных пакетов. + +@item +У @command{nncp-rm} команды появилась @option{-all} опция, применяемая +ко всем нодам сразу. + +@item +У @command{nncp-check} команды появилась @option{-cycle} опция, вводящая +проверку в бесконечный цикл. + +@item +У @command{nncp-rm} команды можно указывать alias-ы имён нод. + +@item +@command{nncp-pkt} может парсить @file{.hdr} файлы. + +@item +Появилась возможность мультивещательной (multicast) рассылки пакетов. +Реализовано всего лишь дополнительным типом простых пакетов и изменением +@command{nncp-toss}, @command{nncp-file} и @command{nncp-exec} команд. + +@end itemize + @node Релиз 7.0.0 @subsection Релиз 7.0.0 @itemize +@item +Минимальная требуемая версия Go 1.13. + @item Хэширование с BLAKE3 на базе деревьев Меркле (Merkle Tree Hashing, MTH) используется вместо BLAKE2b. Из-за этого, обратно @strong{несовместимое} diff --git a/doc/news.texi b/doc/news.texi index cc4eba9..ebfd8c7 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -3,10 +3,48 @@ See also this page @ref{Новости, on russian}. -@node Release 7.0.0 +@node Release 7_1_0 +@section Release 7.1.0 +@itemize + +@item +Fixed workability of @command{nncp-file} and @command{nncp-exec} +commands, that use temporary file (stdin and @option{-use-tmp}). + +@item +Fixed disappearing bad return code in @command{nncp-exec} command. + +@item +Fixed invalid @file{.hdr} generation when transitional packets are used. + +@item +@option{-all} option appeared in @command{nncp-rm} command, applying to +all the nodes at once. + +@item +@option{-cycle} option appeared in @command{nncp-check} command, looping +the check in infinite cycle. + +@item +@command{nncp-rm} command can take node alias name. + +@item +@command{nncp-pkt} can parse @file{.hdr} files. + +@item +Multicasting areas feature appeared. Implemented merely by an additional +plain packet type with @command{nncp-toss}, @command{nncp-file} and +@command{nncp-exec} commands modification. + +@end itemize + +@node Release 7_0_0 @section Release 7.0.0 @itemize +@item +Minimal required Go version 1.13. + @item Merkle Tree-based Hashing with BLAKE3 (MTH) is used instead of BLAKE2b. Because of that, there are backward @strong{incompatible} changes of @@ -46,7 +84,7 @@ Updated dependencies. @end itemize -@node Release 6.6.0 +@node Release 6_6_0 @section Release 6.6.0 @itemize @@ -61,7 +99,7 @@ in local area network, so called MCD (MultiCast Discovery). @end itemize -@node Release 6.5.0 +@node Release 6_5_0 @section Release 6.5.0 @itemize @@ -81,7 +119,7 @@ Kill all packet transmission progress bars in @command{nncp-daemon}, @end itemize -@node Release 6.4.0 +@node Release 6_4_0 @section Release 6.4.0 @itemize @@ -90,7 +128,7 @@ Fixed possible race in online protocol, that lead to panic. @end itemize -@node Release 6.3.0 +@node Release 6_3_0 @section Release 6.3.0 @itemize @@ -99,7 +137,7 @@ Fixed possible panic while showing progress during online protocol. @end itemize -@node Release 6.2.1 +@node Release 6_2_1 @section Release 6.2.1 @itemize @@ -108,7 +146,7 @@ Three places in logs contained excess @code{%s}. @end itemize -@node Release 6.2.0 +@node Release 6_2_0 @section Release 6.2.0 @itemize @@ -121,7 +159,7 @@ Should be no visible differences to the end user. @end itemize -@node Release 6.1.0 +@node Release 6_1_0 @section Release 6.1.0 @itemize @@ -150,7 +188,7 @@ listing on filesystems with big block's size. @end itemize -@node Release 6.0.0 +@node Release 6_0_0 @section Release 6.0.0 @itemize @@ -175,7 +213,7 @@ expectations of specified cron expression. @end itemize -@node Release 5.6.0 +@node Release 5_6_0 @section Release 5.6.0 @itemize @@ -197,7 +235,7 @@ supports modules. @end itemize -@node Release 5.5.1 +@node Release 5_5_1 @section Release 5.5.1 @itemize @@ -207,7 +245,7 @@ variables in @file{config} during installation. @end itemize -@node Release 5.5.0 +@node Release 5_5_0 @section Release 5.5.0 @itemize @@ -232,7 +270,7 @@ Updated dependencies. Minimal required Go version is 1.12. @end itemize -@node Release 5.4.1 +@node Release 5_4_1 @section Release 5.4.1 @itemize @@ -241,7 +279,7 @@ Fixed @code{SENDMAIL} variable usage during the build. @end itemize -@node Release 5.4.0 +@node Release 5_4_0 @section Release 5.4.0 @itemize @@ -255,7 +293,7 @@ implementation is included in tarball. @end itemize -@node Release 5.3.3 +@node Release 5_3_3 @section Release 5.3.3 @itemize @@ -267,7 +305,7 @@ Updated dependencies. @end itemize -@node Release 5.3.2 +@node Release 5_3_2 @section Release 5.3.2 @itemize @@ -278,7 +316,7 @@ forcefully disconnect. @end itemize -@node Release 5.3.1 +@node Release 5_3_1 @section Release 5.3.1 @itemize @@ -292,7 +330,7 @@ about each packet in the spool. @end itemize -@node Release 5.3.0 +@node Release 5_3_0 @section Release 5.3.0 @itemize @@ -322,7 +360,7 @@ SP-connection. That allows faster determining of connection unworkability. @end itemize -@node Release 5.2.1 +@node Release 5_2_1 @section Release 5.2.1 @itemize @@ -331,7 +369,7 @@ Fixed SP protocol error handling, sometimes causing program panic. @end itemize -@node Release 5.2.0 +@node Release 5_2_0 @section Release 5.2.0 @itemize @@ -349,7 +387,7 @@ Free disk space check during @command{nncp-bundle -rx} call. @end itemize -@node Release 5.1.2 +@node Release 5_1_2 @section Release 5.1.2 @itemize @@ -367,7 +405,7 @@ Explicit directories fsync-ing for guaranteed files renaming. @end itemize -@node Release 5.1.1 +@node Release 5_1_1 @section Release 5.1.1 @itemize @@ -376,7 +414,7 @@ Fixed workability of @command{nncp-file} with @option{-chunked 0} option. @end itemize -@node Release 5.1.0 +@node Release 5_1_0 @section Release 5.1.0 @itemize @@ -402,7 +440,7 @@ Ability to notify about successfully executed commands (exec) with @end itemize -@node Release 5.0.0 +@node Release 5_0_0 @section Release 5.0.0 @itemize @@ -450,13 +488,13 @@ Forbid any later GNU GPL version autousage @end itemize -@node Release 4.1 +@node Release 4_1 @section Release 4.1 @itemize @item Workability on GNU/Linux systems and Go 1.10 is fixed. @end itemize -@node Release 4.0 +@node Release 4_0 @section Release 4.0 @itemize @@ -492,13 +530,13 @@ Begin using of @code{go.mod} subsystem. @end itemize -@node Release 3.4 +@node Release 3_4 @section Release 3.4 @itemize @item @command{nncp-daemon} can be run as @command{inetd}-service. @end itemize -@node Release 3.3 +@node Release 3_3 @section Release 3.3 @itemize @@ -528,7 +566,7 @@ for @command{nncp-file} from 196 to 224. @end itemize -@node Release 3.2 +@node Release 3_2 @section Release 3.2 @itemize @item @@ -540,14 +578,14 @@ inside. NNCP accidentally was dependant on that bug. Explicit adding of archives. @end itemize -@node Release 3.1 +@node Release 3_1 @section Release 3.1 @itemize @item Ability to disable relaying at all using @verb{|-via -|} command line option. @end itemize -@node Release 3.0 +@node Release 3_0 @section Release 3.0 @itemize @@ -600,7 +638,7 @@ off by specifying zero value. @end itemize -@node Release 2.0 +@node Release 2_0 @section Release 2.0 @itemize @@ -619,7 +657,7 @@ identical to BLAKE2). @end itemize -@node Release 1.0 +@node Release 1_0 @section Release 1.0 @itemize @@ -653,19 +691,19 @@ command line argument, or environment variable. @end itemize -@node Release 0.12 +@node Release 0_12 @section Release 0.12 @itemize @item Sendmail command is called with @env{NNCP_SENDER} environment variable. @end itemize -@node Release 0.11 +@node Release 0_11 @section Release 0.11 @itemize @item @command{nncp-stat}'s command output is sorted by node name. @end itemize -@node Release 0.10 +@node Release 0_10 @section Release 0.10 @itemize @item @@ -673,7 +711,7 @@ command line argument, or environment variable. @file{SRC} path's element will be used by default. @end itemize -@node Release 0.9 +@node Release 0_9 @section Release 0.9 @itemize @item @@ -681,7 +719,7 @@ Fix @option{-rx}/@option{-tx} arguments processing in @command{nncp-call} command. They were ignored. @end itemize -@node Release 0.8 +@node Release 0_8 @section Release 0.8 @itemize @item @@ -689,7 +727,7 @@ Little bugfix in @command{nncp-file} command, where @option{-minsize} option for unchunked transfer was not in KiBs, but in bytes. @end itemize -@node Release 0.7 +@node Release 0_7 @section Release 0.7 @itemize @@ -729,14 +767,14 @@ Cryptographic libraries (dependencies) are updated. @end itemize -@node Release 0.6 +@node Release 0_6 @section Release 0.6 @itemize @item Small @command{nncp-rm} command appeared. @item Cryptographic libraries (dependencies) are updated. @end itemize -@node Release 0.5 +@node Release 0_5 @section Release 0.5 @itemize @item @@ -744,7 +782,7 @@ Trivial small fix in default niceness level of @command{nncp-file} and @command{nncp-freq} commands. @end itemize -@node Release 0.4 +@node Release 0_4 @section Release 0.4 @itemize @@ -764,13 +802,13 @@ is useful during @command{nncp-xfer} usage. @end itemize -@node Release 0.3 +@node Release 0_3 @section Release 0.3 @itemize @item Fixed compatibility with Go 1.6. @end itemize -@node Release 0.2 +@node Release 0_2 @section Release 0.2 @itemize diff --git a/doc/pkt/area.texi b/doc/pkt/area.texi new file mode 100644 index 0000000..172c09c --- /dev/null +++ b/doc/pkt/area.texi @@ -0,0 +1,23 @@ +@node Encrypted area +@section Encrypted area packet + +@ref{Multicast} area messages contains the encrypted packet, that is +completely similar and have the same format as an ordinary +@ref{Encrypted, encrypted packet}. But instead of the node's identity, +area's identity is used as a recipient. + +For example when @code{nodeA} sends multicast packet with file +transmission and @code{nodeB} is the area's subscriber, then +@code{nodeA} has an encrypted packet to the @code{nodeB} in the outgoing +spool directory: + +@verbatim +ENCRYPTED PACKET (sender=nodeA, recipient=nodeB) WRAPS + PLAIN PACKET (type=area, path=AREA ID) WRAPS + ENCRYPTED PACKET (sender=nodeA, recipient=AREA) WRAPS <-- MsgHash + PLAIN PACKET (type=file, path=FILENAME) WRAPS + FILE CONTENTS +@end verbatim + +Area's message identity is the BLAKE2b-256 hash of header of the area's +packet encrypted packet. diff --git a/doc/pkt.texi b/doc/pkt/encrypted.texi similarity index 61% rename from doc/pkt.texi rename to doc/pkt/encrypted.texi index b479113..b036147 100644 --- a/doc/pkt.texi +++ b/doc/pkt/encrypted.texi @@ -1,76 +1,3 @@ -@node Packet -@unnumbered Packet format - -All packets are -@url{https://tools.ietf.org/html/rfc4506, XDR}-encoded structures. - -@menu -* Plain packet: Plain. -* Encrypted packet: Encrypted. -@end menu - -@node Plain -@section Plain packet - -Plain packet contains either the whole file, or file request (freq), or -transition packet or exec message. It is called "plain", because it -contains plaintext, but plain packets would never be stored on your hard -drive. - -@verbatim - HEADER -+--------------------------------------+--...---+ -| MAGIC | TYPE | NICE | PATHLEN | PATH | PAYLOAD| -+--------------------------------------+--...---+ -@end verbatim - -@multitable @columnfractions 0.2 0.3 0.5 -@headitem @tab XDR type @tab Value -@item Magic number @tab - 8-byte, fixed length opaque data @tab - @verb{|N N C P P 0x00 0x00 0x03|} -@item Payload type @tab - unsigned integer @tab - 0 (file), 1 (freq), 2 (exec), 3 (transition), 4 (exec-fat) -@item Niceness @tab - unsigned integer @tab - 1-255, preferred packet @ref{Niceness, niceness} level -@item Path length @tab - unsigned integer @tab - actual length of @emph{path} field's payload -@item Path @tab - 255 byte, fixed length opaque data @tab - @itemize - @item UTF-8 encoded destination path for file transfer - @item UTF-8 encoded source path for file request - @item UTF-8 encoded, zero byte separated, exec's arguments - @item Node's id the transition packet must be relayed on - @end itemize -@end multitable - -Path has fixed size because of hiding its actual length -- it is -valuable metadata. Payload is appended to the header -- it is not stored -as XDR field, because XDR has no ability to pass more than 4 GiB of -opaque data. Moreover most XDR libraries store fields in the memory in -practice. - -Depending on the packet's type, payload could store: - -@itemize -@item File contents -@item Destination path for freq -@item @url{https://facebook.github.io/zstd/, Zstandard} compressed exec body -@item Whole encrypted packet we need to relay on -@item Uncompressed exec body -@end itemize - -Also depending on packet's type, niceness level means: - -@itemize -@item Preferable niceness level for files sent by freq -@item @env{NNCP_NICE} variable's value passed during @ref{CfgExec} invocation. -@end itemize - @node Encrypted @section Encrypted packet diff --git a/doc/pkt/index.texi b/doc/pkt/index.texi new file mode 100644 index 0000000..7d29946 --- /dev/null +++ b/doc/pkt/index.texi @@ -0,0 +1,15 @@ +@node Packet +@unnumbered Packet format + +All packets are +@url{https://tools.ietf.org/html/rfc4506, XDR}-encoded structures. + +@menu +* Plain packet: Plain. +* Encrypted packet: Encrypted. +* Encrypted area packet: Encrypted area. +@end menu + +@include pkt/plain.texi +@include pkt/encrypted.texi +@include pkt/area.texi diff --git a/doc/pkt/plain.texi b/doc/pkt/plain.texi new file mode 100644 index 0000000..954b893 --- /dev/null +++ b/doc/pkt/plain.texi @@ -0,0 +1,144 @@ +@node Plain +@section Plain packet + +Plain packet contains either the whole file, or file request (freq), or +transition packet or exec message. It is called "plain", because it +contains plaintext, but plain packets would never be stored on your hard +drive. + +@verbatim + HEADER ++--------------------------------------+--...---+ +| MAGIC | TYPE | NICE | PATHLEN | PATH | PAYLOAD| ++--------------------------------------+--...---+ +@end verbatim + +@multitable @columnfractions 0.2 0.3 0.5 +@headitem @tab XDR type @tab Value +@item Magic number @tab + 8-byte, fixed length opaque data @tab + @verb{|N N C P P 0x00 0x00 0x03|} +@item Payload type @tab + unsigned integer @tab + @enumerate 0 + @item file (file transmission) + @item freq (file request) + @item exec (compressed exec) + @item trns (transition) + @item exec-fat (uncompressed exec) + @item area (@ref{Multicast, multicast} area message) + @end enumerate +@item Niceness @tab + unsigned integer @tab + 1-255, preferred packet @ref{Niceness, niceness} level +@item Path length @tab + unsigned integer @tab + actual length of @emph{path} field's payload +@item Path @tab + 255 byte, fixed length opaque data @tab + Depending on packet's type, path holds: + @itemize + @item UTF-8 encoded destination path for file transfer + @item UTF-8 encoded source path for file request + @item UTF-8 encoded, zero byte separated, exec's arguments + @item Node's id the transition packet must be relayed on + @item Multicast area's id + @end itemize +@end multitable + +Path has fixed size because of hiding its actual length -- it is +valuable metadata. Payload is appended to the header -- it is not stored +as XDR field, because XDR has no ability to pass more than 4 GiB of +opaque data. Moreover most XDR libraries store fields in the memory in +practice. + +Depending on the packet's type, payload could store: + +@itemize +@item File contents +@item Destination path for freq +@item Optionally @url{https://facebook.github.io/zstd/, Zstandard} + compressed exec body +@item Whole encrypted packet we need to relay on +@item Multicast area message wrap with another encrypted packet inside +@end itemize + +Also depending on packet's type, niceness level means: + +@itemize +@item Preferable niceness level for files sent by freq +@item @env{NNCP_NICE} variable's value passed during @ref{CfgExec} invocation. +@end itemize + +So plain packets can hold following paths and payloads: + +@table @code + +@item file +@example + +--------------- PATH ---------------+ +---- PAYLOAD ---+ + / \ / \ ++----------------------------------------+---------------...--+ +| FILENAME | 0x00 ... variable ... 0x00 | FILE CONTENTS | ++----------------------------------------+---------------...--+ + \ / + PATHLEN +@end example + +@item freq +@example + +--------------- PATH ---------------+ +---- PAYLOAD ---+ + / \ / \ ++----------------------------------------+---------------...--+ +| FILENAME | 0x00 ... variable ... 0x00 | FILENAME | ++----------------------------------------+---------------...--+ + \ / + PATHLEN +@end example + +@item exec +@example + +----------------------- PATH -------------------------+ +---- PAYLOAD ---+ + / \ / \ ++----------------------------------------------------------+---------------...--+ +| HANDLE | ARG0 0x00 ARG1 ...| 0x00 ... variable ... 0x00 | ZSTD DATA | ++----------------------------------------------------------+---------------...--+ + \ / + +-------- PATHLEN --------+ +@end example + +@item exec-fat +@example + +----------------------- PATH -------------------------+ +---- PAYLOAD ---+ + / \ / \ ++----------------------------------------------------------+---------------...--+ +| HANDLE | ARG0 0x00 ARG1 ...| 0x00 ... variable ... 0x00 | DATA | ++----------------------------------------------------------+---------------...--+ + \ / + +-------- PATHLEN --------+ +@end example + +@item trns +@example + +------- PATH ---------+ +---- PAYLOAD ---+ + / \ / \ ++--------------------------+---------------...--+ +| NODE ID | 0x00 ... 0x00 | ENCRYPTED PACKET | ++--------------------------+---------------...--+ + \ / + PATHLEN +@end example + +@item area +@example + +------- PATH ---------+ +---- PAYLOAD ---+ + / \ / \ ++--------------------------+---------------...--+ +| AREA ID | 0x00 ... 0x00 | ENCRYPTED PACKET | ++--------------------------+---------------...--+ + \ / + PATHLEN +@end example +See also @ref{Encrypted area, encrypted area packet}. + +@end table diff --git a/makedist.sh b/makedist.sh index 96946f1..75b50e2 100755 --- a/makedist.sh +++ b/makedist.sh @@ -120,9 +120,10 @@ size=$(( $(stat -f %z $tarball) / 1024 )) hash=$(gpg --print-md SHA256 < $tarball) release_date=$(date "+%Y-%m-%d") +release_underscored=`echo $release | tr . _` cat < + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package nncp + +import ( + "errors" +) + +const AreaDir = "area" + +var ( + PktAreaOverhead int64 +) + +type AreaId [32]byte + +func (id AreaId) String() string { + return Base32Codec.EncodeToString(id[:]) +} + +type Area struct { + Name string + Id *AreaId + Pub *[32]byte + Prv *[32]byte + + Subs []*NodeId + + Exec map[string][]string + Incoming *string + + AllowUnknown bool +} + +func AreaIdFromString(raw string) (*AreaId, error) { + idRaw, err := Base32Codec.DecodeString(raw) + if err != nil { + return nil, err + } + if len(idRaw) != 32 { + return nil, errors.New("Invalid area id size") + } + areaId := new(AreaId) + copy(areaId[:], idRaw) + return areaId, nil +} + +func (ctx *Ctx) AreaName(id *AreaId) string { + area := ctx.AreaId2Area[*id] + if area == nil { + return id.String() + } + return area.Name +} diff --git a/src/call.go b/src/call.go index 648190f..1e20aa2 100644 --- a/src/call.go +++ b/src/call.go @@ -45,6 +45,7 @@ type Call struct { AutoTossNoFreq bool AutoTossNoExec bool AutoTossNoTrns bool + AutoTossNoArea bool } func (ctx *Ctx) CallNode( @@ -110,7 +111,7 @@ func (ctx *Ctx) CallNode( node.Name, int(state.Duration.Hours()), int(state.Duration.Minutes()), - int(state.Duration.Seconds()/60), + int(state.Duration.Seconds())%60, humanize.IBytes(uint64(state.RxBytes)), humanize.IBytes(uint64(state.RxSpeed)), humanize.IBytes(uint64(state.TxBytes)), diff --git a/src/cfg.go b/src/cfg.go index bb101ca..bacfd7e 100644 --- a/src/cfg.go +++ b/src/cfg.go @@ -91,6 +91,7 @@ type CallJSON struct { AutoTossNoFreq *bool `json:"autotoss-nofreq,omitempty"` AutoTossNoExec *bool `json:"autotoss-noexec,omitempty"` AutoTossNoTrns *bool `json:"autotoss-notrns,omitempty"` + AutoTossNoArea *bool `json:"autotoss-noarea,omitempty"` } type NodeOurJSON struct { @@ -114,6 +115,19 @@ type NotifyJSON struct { Exec map[string]*FromToJSON `json:"exec,omitempty"` } +type AreaJSON struct { + Id string `json:"id"` + Pub string `json:"pub"` + Prv *string `json:"prv,omitempty"` + + Subs []string `json:"subs"` + + Exec map[string][]string `json:"exec,omitempty"` + Incoming *string `json:"incoming,omitempty"` + + AllowUnknown *bool `json:"allow-unknown,omitempty"` +} + type CfgJSON struct { Spool string `json:"spool"` Log string `json:"log"` @@ -129,6 +143,8 @@ type CfgJSON struct { MCDRxIfis []string `json:"mcd-listen"` MCDTxIfis map[string]int `json:"mcd-send"` + + Areas map[string]AreaJSON `json:"areas"` } func NewNode(name string, cfg NodeJSON) (*Node, error) { @@ -314,6 +330,9 @@ func NewNode(name string, cfg NodeJSON) (*Node, error) { if callCfg.AutoTossNoTrns != nil { call.AutoTossNoTrns = *callCfg.AutoTossNoTrns } + if callCfg.AutoTossNoArea != nil { + call.AutoTossNoArea = *callCfg.AutoTossNoArea + } calls = append(calls, &call) } @@ -414,6 +433,52 @@ func NewNodeOur(cfg *NodeOurJSON) (*NodeOur, error) { return &node, nil } +func NewArea(ctx *Ctx, name string, cfg *AreaJSON) (*Area, error) { + areaId, err := AreaIdFromString(cfg.Id) + if err != nil { + return nil, err + } + subs := make([]*NodeId, 0, len(cfg.Subs)) + for _, s := range cfg.Subs { + node, err := ctx.FindNode(s) + if err != nil { + return nil, err + } + subs = append(subs, node.Id) + } + area := Area{ + Name: name, + Id: areaId, + Pub: new([32]byte), + Subs: subs, + Exec: cfg.Exec, + Incoming: cfg.Incoming, + } + pub, err := Base32Codec.DecodeString(cfg.Pub) + if err != nil { + return nil, err + } + if len(pub) != 32 { + return nil, errors.New("Invalid pub size") + } + copy(area.Pub[:], pub) + if cfg.Prv != nil { + prv, err := Base32Codec.DecodeString(*cfg.Prv) + if err != nil { + return nil, err + } + if len(prv) != 32 { + return nil, errors.New("Invalid prv size") + } + area.Prv = new([32]byte) + copy(area.Prv[:], prv) + } + if cfg.AllowUnknown != nil { + area.AllowUnknown = *cfg.AllowUnknown + } + return &area, nil +} + func CfgParse(data []byte) (*Ctx, error) { var err error if bytes.Compare(data[:8], MagicNNCPBv3.B[:]) == 0 { @@ -528,5 +593,15 @@ func CfgParse(data []byte) (*Ctx, error) { ) } } + ctx.AreaId2Area = make(map[AreaId]*Area, len(cfgJSON.Areas)) + ctx.AreaName2Id = make(map[string]*AreaId, len(cfgJSON.Areas)) + for name, areaJSON := range cfgJSON.Areas { + area, err := NewArea(&ctx, name, &areaJSON) + if err != nil { + return nil, err + } + ctx.AreaId2Area[*area.Id] = area + ctx.AreaName2Id[name] = area.Id + } return &ctx, nil } diff --git a/src/cmd/nncp-call/main.go b/src/cmd/nncp-call/main.go index 5d632d4..09c3e18 100644 --- a/src/cmd/nncp-call/main.go +++ b/src/cmd/nncp-call/main.go @@ -66,6 +66,7 @@ func main() { autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") + autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -174,6 +175,7 @@ func main() { *autoTossNoFreq, *autoTossNoExec, *autoTossNoTrns, + *autoTossNoArea, ) } diff --git a/src/cmd/nncp-caller/main.go b/src/cmd/nncp-caller/main.go index 4608484..d1a5b34 100644 --- a/src/cmd/nncp-caller/main.go +++ b/src/cmd/nncp-caller/main.go @@ -56,6 +56,7 @@ func main() { autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") + autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -200,6 +201,7 @@ func main() { call.AutoTossNoFreq || *autoTossNoFreq, call.AutoTossNoExec || *autoTossNoExec, call.AutoTossNoTrns || *autoTossNoTrns, + call.AutoTossNoArea || *autoTossNoArea, ) } diff --git a/src/cmd/nncp-cfgnew/main.go b/src/cmd/nncp-cfgnew/main.go index f992601..74f9ae6 100644 --- a/src/cmd/nncp-cfgnew/main.go +++ b/src/cmd/nncp-cfgnew/main.go @@ -19,11 +19,17 @@ along with this program. If not, see . package main import ( + "crypto/rand" + "encoding/json" "flag" "fmt" "log" "os" + "github.com/hjson/hjson-go" + "golang.org/x/crypto/blake2b" + "golang.org/x/crypto/nacl/box" + "go.cypherpunks.ru/nncp/v7" ) @@ -35,6 +41,7 @@ func usage() { func main() { var ( + areaName = flag.String("area", "", "Generate area's keypairs") noComments = flag.Bool("nocomments", false, "Do not include descriptive comments") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") @@ -50,9 +57,75 @@ func main() { fmt.Println(nncp.VersionGet()) return } + if *areaName != "" { + pub, prv, err := box.GenerateKey(rand.Reader) + if err != nil { + log.Fatalln(err) + } + areaId := nncp.AreaId(blake2b.Sum256(pub[:])) + var cfgRaw string + if *noComments { + cfgRaw = fmt.Sprintf(`areas: { + %s: { + id: %s + # KEEP AWAY keypair from the nodes you want only participate in multicast + pub: %s + prv: %s + } +}`, + *areaName, + areaId.String(), + nncp.Base32Codec.EncodeToString(pub[:]), + nncp.Base32Codec.EncodeToString(prv[:]), + ) + } else { + cfgRaw = fmt.Sprintf(`areas: { + %s: { + id: %s + + # KEEP AWAY keypair from the nodes you want only participate in multicast + pub: %s + prv: %s + + # List of subscribers you should multicast area messages to + # subs: ["alice"] + + # Allow incoming files (from the area) saving in that directory + # incoming: /home/areas/%s/incoming + + # Allow incoming area commands execution + # exec: {sendmail: ["%s"]} + + # Allow unknown sender's message tossing (relaying will be made anyway) + # allow-unknown: true + } +}`, + *areaName, + areaId.String(), + nncp.Base32Codec.EncodeToString(pub[:]), + nncp.Base32Codec.EncodeToString(prv[:]), + *areaName, + nncp.DefaultSendmailPath, + ) + } + var cfgGeneral map[string]interface{} + if err = hjson.Unmarshal([]byte(cfgRaw), &cfgGeneral); err != nil { + panic(err) + } + marshaled, err := json.Marshal(cfgGeneral) + if err != nil { + panic(err) + } + var areas map[string]nncp.AreaJSON + if err = json.Unmarshal(marshaled, &areas); err != nil { + panic(err) + } + fmt.Println(cfgRaw) + return + } nodeOur, err := nncp.NewNodeGenerate() if err != nil { - panic(err) + log.Fatalln(err) } var cfgRaw string if *noComments { diff --git a/src/cmd/nncp-check/main.go b/src/cmd/nncp-check/main.go index 6575ded..b2a931c 100644 --- a/src/cmd/nncp-check/main.go +++ b/src/cmd/nncp-check/main.go @@ -24,6 +24,7 @@ import ( "log" "os" "path/filepath" + "time" "go.cypherpunks.ru/nncp/v7" ) @@ -38,6 +39,7 @@ func usage() { func main() { var ( nock = flag.Bool("nock", false, "Process .nock files") + cycle = flag.Uint("cycle", 0, "Repeat check after N seconds in infinite loop") cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") nodeRaw = flag.String("node", "", "Process only that node") spoolPath = flag.String("spool", "", "Override path to spool") @@ -83,6 +85,7 @@ func main() { } } +Cycle: isBad := false for nodeId, node := range ctx.Neigh { if nodeOnly != nil && nodeId != *nodeOnly.Id { @@ -105,6 +108,10 @@ func main() { isBad = true } } + if *cycle > 0 { + time.Sleep(time.Duration(*cycle) * time.Second) + goto Cycle + } if isBad { os.Exit(1) } diff --git a/src/cmd/nncp-daemon/main.go b/src/cmd/nncp-daemon/main.go index 39f2bba..b0628fe 100644 --- a/src/cmd/nncp-daemon/main.go +++ b/src/cmd/nncp-daemon/main.go @@ -105,7 +105,7 @@ func performSP( state.Node.Name, int(state.Duration.Hours()), int(state.Duration.Minutes()), - int(state.Duration.Seconds()/60), + int(state.Duration.Seconds())%60, humanize.IBytes(uint64(state.RxBytes)), humanize.IBytes(uint64(state.RxSpeed)), humanize.IBytes(uint64(state.TxBytes)), @@ -157,6 +157,7 @@ func main() { autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") + autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -208,6 +209,7 @@ func main() { *autoTossNoFreq, *autoTossNoExec, *autoTossNoTrns, + *autoTossNoArea, ) } <-nodeIdC // call completion @@ -273,6 +275,7 @@ func main() { *autoTossNoFreq, *autoTossNoExec, *autoTossNoTrns, + *autoTossNoArea, ) } <-nodeIdC // call completion diff --git a/src/cmd/nncp-exec/main.go b/src/cmd/nncp-exec/main.go index c379ef0..11a9907 100644 --- a/src/cmd/nncp-exec/main.go +++ b/src/cmd/nncp-exec/main.go @@ -24,6 +24,7 @@ import ( "fmt" "log" "os" + "strings" "go.cypherpunks.ru/nncp/v7" ) @@ -31,7 +32,9 @@ import ( func usage() { fmt.Fprintf(os.Stderr, nncp.UsageHeader()) fmt.Fprintf(os.Stderr, "nncp-exec -- send execution command\n\n") - fmt.Fprintf(os.Stderr, "Usage: %s [options] NODE HANDLE [ARG0 ARG1 ...]\nOptions:\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [options] NODE HANDLE [ARG0 ARG1 ...]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] %s:AREA HANDLE [ARG0 ARG1 ...]\nOptions:\n", + os.Args[0], nncp.AreaDir) flag.PrintDefaults() } @@ -93,9 +96,19 @@ func main() { log.Fatalln("Config lacks private keys") } - node, err := ctx.FindNode(flag.Arg(0)) - if err != nil { - log.Fatalln("Invalid NODE specified:", err) + var areaId *nncp.AreaId + var node *nncp.Node + if strings.HasPrefix(flag.Arg(0), nncp.AreaDir+":") { + areaId = ctx.AreaName2Id[flag.Arg(0)[len(nncp.AreaDir)+1:]] + if areaId == nil { + log.Fatalln("Unknown area specified") + } + node = ctx.Neigh[*ctx.SelfId] + } else { + node, err = ctx.FindNode(flag.Arg(0)) + if err != nil { + log.Fatalln("Invalid NODE specified:", err) + } } nncp.ViaOverride(*viaOverride, ctx, node) @@ -111,6 +124,7 @@ func main() { int64(*minSize)*1024, *useTmp, *noCompress, + areaId, ); err != nil { log.Fatalln(err) } diff --git a/src/cmd/nncp-file/main.go b/src/cmd/nncp-file/main.go index 6fbbdc9..6870aec 100644 --- a/src/cmd/nncp-file/main.go +++ b/src/cmd/nncp-file/main.go @@ -31,7 +31,9 @@ import ( func usage() { fmt.Fprintf(os.Stderr, nncp.UsageHeader()) fmt.Fprintf(os.Stderr, "nncp-file -- send file\n\n") - fmt.Fprintf(os.Stderr, "Usage: %s [options] SRC NODE:[DST]\nOptions:\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [options] SRC NODE:[DST]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] SRC %s:AREA:[DST]\nOptions:\n", + os.Args[0], nncp.AreaDir) flag.PrintDefaults() fmt.Fprint(os.Stderr, ` If SRC equals to -, then read data from stdin to temporary file. @@ -93,14 +95,30 @@ func main() { log.Fatalln("Config lacks private keys") } - splitted := strings.SplitN(flag.Arg(1), ":", 2) - if len(splitted) != 2 { + splitted := strings.Split(flag.Arg(1), ":") + if len(splitted) < 2 { usage() os.Exit(1) } - node, err := ctx.FindNode(splitted[0]) - if err != nil { - log.Fatalln("Invalid NODE specified:", err) + var areaId *nncp.AreaId + var node *nncp.Node + if splitted[0] == nncp.AreaDir { + if len(splitted) < 3 { + usage() + os.Exit(1) + } + areaId = ctx.AreaName2Id[splitted[1]] + if areaId == nil { + log.Fatalln("Unknown area specified") + } + node = ctx.Neigh[*ctx.SelfId] + splitted = splitted[2:] + } else { + node, err = ctx.FindNode(splitted[0]) + if err != nil { + log.Fatalln("Invalid NODE specified:", err) + } + splitted = splitted[1:] } nncp.ViaOverride(*viaOverride, ctx, node) @@ -127,10 +145,11 @@ func main() { node, nice, flag.Arg(0), - splitted[1], + strings.Join(splitted, ":"), chunkSize, minSize, nncp.MaxFileSize, + areaId, ); err != nil { log.Fatalln(err) } diff --git a/src/cmd/nncp-pkt/main.go b/src/cmd/nncp-pkt/main.go index 49c23a2..3cd59ec 100644 --- a/src/cmd/nncp-pkt/main.go +++ b/src/cmd/nncp-pkt/main.go @@ -40,7 +40,7 @@ func usage() { fmt.Fprintln(os.Stderr, "Packet is read from stdin.") } -func doPlain(pkt nncp.Pkt, dump, decompress bool) { +func doPlain(ctx *nncp.Ctx, pkt nncp.Pkt, dump, decompress bool) { if dump { bufW := bufio.NewWriter(os.Stdout) var r io.Reader @@ -68,22 +68,30 @@ func doPlain(pkt nncp.Pkt, dump, decompress bool) { payloadType = "file request" case nncp.PktTypeExec: payloadType = "exec compressed" - case nncp.PktTypeExecFat: - payloadType = "exec uncompressed" case nncp.PktTypeTrns: payloadType = "transitional" + case nncp.PktTypeExecFat: + payloadType = "exec uncompressed" + case nncp.PktTypeArea: + payloadType = "area" } var path string switch pkt.Type { case nncp.PktTypeExec, nncp.PktTypeExecFat: path = string(bytes.Replace( - pkt.Path[:pkt.PathLen], - []byte{0}, - []byte(" "), - -1, + pkt.Path[:pkt.PathLen], []byte{0}, []byte(" "), -1, )) case nncp.PktTypeTrns: path = nncp.Base32Codec.EncodeToString(pkt.Path[:pkt.PathLen]) + node, err := ctx.FindNode(path) + if err != nil { + path = fmt.Sprintf("%s (%s)", path, node.Name) + } + case nncp.PktTypeArea: + path = nncp.Base32Codec.EncodeToString(pkt.Path[:pkt.PathLen]) + if areaId, err := nncp.AreaIdFromString(path); err == nil { + path = fmt.Sprintf("%s (%s)", path, ctx.AreaName(areaId)) + } default: path = string(pkt.Path[:pkt.PathLen]) } @@ -94,31 +102,62 @@ func doPlain(pkt nncp.Pkt, dump, decompress bool) { return } -func doEncrypted(pktEnc nncp.PktEnc, dump bool, cfgPath string, beginning []byte) { +func doEncrypted( + ctx *nncp.Ctx, + pktEnc nncp.PktEnc, + dump bool, + beginning []byte, +) { + senderName := "unknown" + senderNode := ctx.Neigh[*pktEnc.Sender] + if senderNode != nil { + senderName = senderNode.Name + } + + recipientName := "unknown" + var area *nncp.Area + recipientNode := ctx.Neigh[*pktEnc.Recipient] + if recipientNode == nil { + area = ctx.AreaId2Area[nncp.AreaId(*pktEnc.Recipient)] + recipientName = "area " + area.Name + } else { + recipientName = recipientNode.Name + } + if !dump { - fmt.Printf( - "Packet type: encrypted\nNiceness: %s (%d)\nSender: %s\nRecipient: %s\n", - nncp.NicenessFmt(pktEnc.Nice), pktEnc.Nice, pktEnc.Sender, pktEnc.Recipient, + fmt.Printf(`Packet type: encrypted +Niceness: %s (%d) +Sender: %s (%s) +Recipient: %s (%s) +`, + nncp.NicenessFmt(pktEnc.Nice), pktEnc.Nice, + pktEnc.Sender, senderName, + pktEnc.Recipient, recipientName, ) return } - ctx, err := nncp.CtxFromCmdline(cfgPath, "", "", false, false, false, false) - if err != nil { - log.Fatalln("Error during initialization:", err) - } if ctx.Self == nil { log.Fatalln("Config lacks private keys") } bufW := bufio.NewWriter(os.Stdout) - if _, _, err = nncp.PktEncRead( - ctx.Self, - ctx.Neigh, - io.MultiReader( - bytes.NewReader(beginning), - bufio.NewReader(os.Stdin), - ), - bufW, - ); err != nil { + var err error + if area == nil { + _, _, _, err = nncp.PktEncRead( + ctx.Self, ctx.Neigh, + io.MultiReader(bytes.NewReader(beginning), bufio.NewReader(os.Stdin)), + bufW, senderNode != nil, nil, + ) + } else { + areaNode := nncp.NodeOur{Id: new(nncp.NodeId), ExchPrv: new([32]byte)} + copy(areaNode.Id[:], area.Id[:]) + copy(areaNode.ExchPrv[:], area.Prv[:]) + _, _, _, err = nncp.PktEncRead( + &areaNode, ctx.Neigh, + io.MultiReader(bytes.NewReader(beginning), bufio.NewReader(os.Stdin)), + bufW, senderNode != nil, nil, + ) + } + if err != nil { log.Fatalln(err) } if err = bufW.Flush(); err != nil { @@ -147,6 +186,11 @@ func main() { return } + ctx, err := nncp.CtxFromCmdline(*cfgPath, "", "", false, false, false, false) + if err != nil { + log.Fatalln("Error during initialization:", err) + } + if *overheads { fmt.Printf( "Plain: %d\nEncrypted: %d\nSize: %d\n", @@ -158,24 +202,12 @@ func main() { } beginning := make([]byte, nncp.PktOverhead) - if _, err := io.ReadFull(os.Stdin, beginning); err != nil { + if _, err := io.ReadFull(os.Stdin, beginning[:nncp.PktEncOverhead]); err != nil { log.Fatalln("Not enough data to read") } - var pkt nncp.Pkt - if _, err := xdr.Unmarshal(bytes.NewReader(beginning), &pkt); err == nil { - switch pkt.Magic { - case nncp.MagicNNCPPv1.B: - log.Fatalln(nncp.MagicNNCPPv1.TooOld()) - case nncp.MagicNNCPPv2.B: - log.Fatalln(nncp.MagicNNCPPv2.TooOld()) - case nncp.MagicNNCPPv3.B: - doPlain(pkt, *dump, *decompress) - return - } - } var pktEnc nncp.PktEnc if _, err := xdr.Unmarshal(bytes.NewReader(beginning), &pktEnc); err == nil { - switch pkt.Magic { + switch pktEnc.Magic { case nncp.MagicNNCPEv1.B: log.Fatalln(nncp.MagicNNCPEv1.TooOld()) case nncp.MagicNNCPEv2.B: @@ -185,7 +217,23 @@ func main() { case nncp.MagicNNCPEv4.B: log.Fatalln(nncp.MagicNNCPEv4.TooOld()) case nncp.MagicNNCPEv5.B: - doEncrypted(pktEnc, *dump, *cfgPath, beginning) + doEncrypted(ctx, pktEnc, *dump, beginning[:nncp.PktEncOverhead]) + return + } + } + + if _, err := io.ReadFull(os.Stdin, beginning[nncp.PktEncOverhead:]); err != nil { + log.Fatalln("Not enough data to read") + } + var pkt nncp.Pkt + if _, err := xdr.Unmarshal(bytes.NewReader(beginning), &pkt); err == nil { + switch pkt.Magic { + case nncp.MagicNNCPPv1.B: + log.Fatalln(nncp.MagicNNCPPv1.TooOld()) + case nncp.MagicNNCPPv2.B: + log.Fatalln(nncp.MagicNNCPPv2.TooOld()) + case nncp.MagicNNCPPv3.B: + doPlain(ctx, pkt, *dump, *decompress) return } } diff --git a/src/cmd/nncp-rm/main.go b/src/cmd/nncp-rm/main.go index f8f4289..39cd0c6 100644 --- a/src/cmd/nncp-rm/main.go +++ b/src/cmd/nncp-rm/main.go @@ -37,12 +37,13 @@ func usage() { fmt.Fprintf(os.Stderr, "nncp-rm -- remove packet\n\n") fmt.Fprintf(os.Stderr, "Usage: %s [options] -tmp\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s [options] -lock\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -part\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -seen\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -nock\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -hdr\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE {-rx|-tx}\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] -node NODE -pkt PKT\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -part\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -seen\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -nock\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -hdr\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -area\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} {-rx|-tx}\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -pkt PKT\n", os.Args[0]) fmt.Fprintln(os.Stderr, "-older option's time units are: (s)econds, (m)inutes, (h)ours, (d)ays") fmt.Fprintln(os.Stderr, "Options:") flag.PrintDefaults() @@ -51,6 +52,7 @@ func usage() { func main() { var ( cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") + doAll = flag.Bool("all", false, "Apply remove rules to all nodes") doTmp = flag.Bool("tmp", false, "Remove all temporary files") doHdr = flag.Bool("hdr", false, "Remove all .hdr files") doLock = flag.Bool("lock", false, "Remove all lock files") @@ -60,6 +62,7 @@ func main() { doPart = flag.Bool("part", false, "Remove only .part files") doSeen = flag.Bool("seen", false, "Remove only .seen files") doNoCK = flag.Bool("nock", false, "Remove only .nock files") + doArea = flag.Bool("area", false, "Remove only area/*.seen files") older = flag.String("older", "", "XXX{smhd}: only older than XXX number of time units") dryRun = flag.Bool("dryrun", false, "Do not actually remove files") pktRaw = flag.String("pkt", "", "Packet to remove") @@ -165,70 +168,117 @@ func main() { } return } + var nodeId *nncp.NodeId if *nodeRaw == "" { - usage() - os.Exit(1) - } - node, err := ctx.FindNode(*nodeRaw) - if err != nil { - log.Fatalln("Invalid -node specified:", err) + if !*doAll { + usage() + os.Exit(1) + } + } else { + node, err := ctx.FindNode(*nodeRaw) + if err != nil { + log.Fatalln("Invalid -node specified:", err) + } + nodeId = node.Id } - remove := func(xx nncp.TRxTx) error { - return filepath.Walk( - filepath.Join(ctx.Spool, node.Id.String(), string(xx)), - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - logMsg := func(les nncp.LEs) string { - return fmt.Sprintf("File %s: removed", path) - } - if now.Sub(info.ModTime()) < oldBoundary { - ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: path}}, func(les nncp.LEs) string { - return fmt.Sprintf("File %s: too fresh, skipping", path) - }) - return nil - } - if (*doSeen && strings.HasSuffix(info.Name(), nncp.SeenSuffix)) || - (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || - (*doHdr && strings.HasSuffix(info.Name(), nncp.HdrSuffix)) || - (*doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix)) { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) - if *dryRun { - return nil + for _, node := range ctx.Neigh { + if nodeId != nil && node.Id != nodeId { + continue + } + remove := func(xx nncp.TRxTx) error { + return filepath.Walk( + filepath.Join(ctx.Spool, node.Id.String(), string(xx)), + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err } - return os.Remove(path) - } - if *pktRaw != "" && filepath.Base(info.Name()) == *pktRaw { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) - if *dryRun { + if info.IsDir() { return nil } - return os.Remove(path) - } - if !*doSeen && !*doNoCK && !*doHdr && !*doPart && - (*doRx || *doTx) && - ((*doRx && xx == nncp.TRx) || (*doTx && xx == nncp.TTx)) { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) - if *dryRun { + logMsg := func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: path}}, func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", path) + }) return nil } - return os.Remove(path) - } - return nil - }) - } - if *pktRaw != "" || *doRx || *doSeen || *doNoCK || *doHdr || *doPart { - if err = remove(nncp.TRx); err != nil { - log.Fatalln("Can not remove:", err) + if (*doSeen && strings.HasSuffix(info.Name(), nncp.SeenSuffix)) || + (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || + (*doHdr && strings.HasSuffix(info.Name(), nncp.HdrSuffix)) || + (*doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix)) { + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + } + if *pktRaw != "" && filepath.Base(info.Name()) == *pktRaw { + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + } + if !*doSeen && !*doNoCK && !*doHdr && !*doPart && + (*doRx || *doTx) && + ((*doRx && xx == nncp.TRx) || (*doTx && xx == nncp.TTx)) { + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + } + return nil + }) } - } - if *pktRaw != "" || *doTx || *doHdr { - if err = remove(nncp.TTx); err != nil { - log.Fatalln("Can not remove:", err) + if *pktRaw != "" || *doRx || *doSeen || *doNoCK || *doHdr || *doPart { + if err = remove(nncp.TRx); err != nil { + log.Fatalln("Can not remove:", err) + } + } + if *pktRaw != "" || *doTx || *doHdr { + if err = remove(nncp.TTx); err != nil { + log.Fatalln("Can not remove:", err) + } + } + if *doArea { + if err = filepath.Walk( + filepath.Join(ctx.Spool, node.Id.String(), nncp.AreaDir), + func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if info.IsDir() { + return nil + } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: path}}, func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", path) + }) + return nil + } + if strings.HasSuffix(info.Name(), nncp.SeenSuffix) { + ctx.LogI( + "rm", + nncp.LEs{{K: "File", V: path}}, + func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + }, + ) + if *dryRun { + return nil + } + return os.Remove(path) + } + return nil + }); err != nil { + log.Fatalln("Can not remove:", err) + } } } } diff --git a/src/cmd/nncp-toss/main.go b/src/cmd/nncp-toss/main.go index d6b82fb..af3a59d 100644 --- a/src/cmd/nncp-toss/main.go +++ b/src/cmd/nncp-toss/main.go @@ -47,6 +47,7 @@ func main() { noFreq = flag.Bool("nofreq", false, "Do not process \"freq\" packets") noExec = flag.Bool("noexec", false, "Do not process \"exec\" packets") noTrns = flag.Bool("notrns", false, "Do not process \"transitional\" packets") + noArea = flag.Bool("noarea", false, "Do not process \"area\" packets") spoolPath = flag.String("spool", "", "Override path to spool") logPath = flag.String("log", "", "Override path to logfile") quiet = flag.Bool("quiet", false, "Print only errors") @@ -106,14 +107,18 @@ Cycle: } isBad = ctx.Toss( node.Id, + nncp.TRx, nice, - *dryRun, - *doSeen, - *noFile, - *noFreq, - *noExec, - *noTrns, - ) + *dryRun, *doSeen, *noFile, *noFreq, *noExec, *noTrns, *noArea, + ) || isBad + if nodeId == *ctx.SelfId { + isBad = ctx.Toss( + node.Id, + nncp.TTx, + nice, + *dryRun, false, true, true, true, true, *noArea, + ) || isBad + } } if *cycle > 0 { time.Sleep(time.Duration(*cycle) * time.Second) diff --git a/src/ctx.go b/src/ctx.go index fb26185..63d1fc2 100644 --- a/src/ctx.go +++ b/src/ctx.go @@ -36,6 +36,9 @@ type Ctx struct { Neigh map[NodeId]*Node Alias map[string]*NodeId + AreaId2Area map[AreaId]*Area + AreaName2Id map[string]*AreaId + Spool string LogPath string UmaskForce *int diff --git a/src/go.mod b/src/go.mod index 3afd292..e84a8f4 100644 --- a/src/go.mod +++ b/src/go.mod @@ -16,4 +16,4 @@ require ( lukechampine.com/blake3 v1.1.5 ) -go 1.12 +go 1.13 diff --git a/src/humanizer.go b/src/humanizer.go index 69d1bb5..701b590 100644 --- a/src/humanizer.go +++ b/src/humanizer.go @@ -25,15 +25,6 @@ import ( "go.cypherpunks.ru/recfile" ) -func (ctx *Ctx) NodeName(id *NodeId) string { - idS := id.String() - node, err := ctx.FindNode(idS) - if err == nil { - return node.Name - } - return idS -} - func (ctx *Ctx) HumanizeRec(rec string) string { r := recfile.NewReader(strings.NewReader(rec)) le, err := r.NextMap() diff --git a/src/jobs.go b/src/jobs.go index 0819738..6e42877 100644 --- a/src/jobs.go +++ b/src/jobs.go @@ -20,6 +20,7 @@ package nncp import ( "bytes" "fmt" + "io" "os" "path/filepath" "strings" @@ -44,9 +45,9 @@ type Job struct { HshValue *[MTHSize]byte } -func (ctx *Ctx) HdrRead(fd *os.File) (*PktEnc, []byte, error) { +func (ctx *Ctx) HdrRead(r io.Reader) (*PktEnc, []byte, error) { var pktEnc PktEnc - _, err := xdr.Unmarshal(fd, &pktEnc) + _, err := xdr.Unmarshal(r, &pktEnc) if err != nil { return nil, nil, err } diff --git a/src/magic.go b/src/magic.go index 2f0835c..938d333 100644 --- a/src/magic.go +++ b/src/magic.go @@ -17,7 +17,10 @@ along with this program. If not, see . package nncp -import "fmt" +import ( + "errors" + "fmt" +) type Magic struct { B [8]byte @@ -26,6 +29,10 @@ type Magic struct { } var ( + MagicNNCPAv1 = Magic{ + B: [8]byte{'N', 'N', 'C', 'P', 'A', 0, 0, 1}, + Name: "NNCPAv1 (area packet v1)", Till: "now", + } MagicNNCPBv1 = Magic{ B: [8]byte{'N', 'N', 'C', 'P', 'B', 0, 0, 1}, Name: "NNCPBv1 (EBlob v1)", Till: "1.0", @@ -86,6 +93,8 @@ var ( B: [8]byte{'N', 'N', 'C', 'P', 'P', 0, 0, 3}, Name: "NNCPPv3 (plain packet v3)", Till: "now", } + + BadMagic error = errors.New("Unknown magic number") ) func (m *Magic) TooOld() error { diff --git a/src/nncp.go b/src/nncp.go index 1edc777..d3ad2a7 100644 --- a/src/nncp.go +++ b/src/nncp.go @@ -40,7 +40,7 @@ along with this program. If not, see .` const Base32Encoded32Len = 52 var ( - Version string = "7.0.0" + Version string = "7.1.0" Base32Codec *base32.Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) ) diff --git a/src/node.go b/src/node.go index c965796..aabf847 100644 --- a/src/node.go +++ b/src/node.go @@ -20,6 +20,7 @@ package nncp import ( "crypto/rand" "errors" + "fmt" "sync" "time" @@ -29,6 +30,8 @@ import ( "golang.org/x/crypto/nacl/box" ) +const DummyB32Id = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + type NodeId [blake2b.Size256]byte func (id NodeId) String() string { @@ -114,13 +117,21 @@ func (nodeOur *NodeOur) Their() *Node { func NodeIdFromString(raw string) (*NodeId, error) { decoded, err := Base32Codec.DecodeString(raw) if err != nil { - return nil, err + return nil, fmt.Errorf("Can not parse node: %s: %s", raw, err) } if len(decoded) != blake2b.Size256 { return nil, errors.New("Invalid node id size") } - buf := new([blake2b.Size256]byte) - copy(buf[:], decoded) - nodeId := NodeId(*buf) - return &nodeId, nil + nodeId := new(NodeId) + copy(nodeId[:], decoded) + return nodeId, nil +} + +func (ctx *Ctx) NodeName(id *NodeId) string { + idS := id.String() + node, err := ctx.FindNode(idS) + if err == nil { + return node.Name + } + return idS } diff --git a/src/pkt.go b/src/pkt.go index 135e850..bd3fb23 100644 --- a/src/pkt.go +++ b/src/pkt.go @@ -44,6 +44,7 @@ const ( PktTypeExec PktType = iota PktTypeTrns PktType = iota PktTypeExecFat PktType = iota + PktTypeArea PktType = iota MaxPathSize = 1<<8 - 1 @@ -53,7 +54,6 @@ const ( ) var ( - BadMagic error = errors.New("Unknown magic number") BadPktType error = errors.New("Unknown packet type") PktOverhead int64 @@ -95,7 +95,7 @@ func init() { PktOverhead = int64(n) buf.Reset() - dummyId, err := NodeIdFromString("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + dummyId, err := NodeIdFromString(DummyB32Id) if err != nil { panic(err) } @@ -273,7 +273,7 @@ func PktEncWrite( return pktEncRaw, nil } -func TbsVerify(our *NodeOur, their *Node, pktEnc *PktEnc) ([]byte, bool, error) { +func TbsPrepare(our *NodeOur, their *Node, pktEnc *PktEnc) []byte { tbs := PktTbs{ Magic: MagicNNCPEv5.B, Nice: pktEnc.Nice, @@ -283,9 +283,14 @@ func TbsVerify(our *NodeOur, their *Node, pktEnc *PktEnc) ([]byte, bool, error) } var tbsBuf bytes.Buffer if _, err := xdr.Marshal(&tbsBuf, &tbs); err != nil { - return nil, false, err + panic(err) } - return tbsBuf.Bytes(), ed25519.Verify(their.SignPub, tbsBuf.Bytes(), pktEnc.Sign[:]), nil + return tbsBuf.Bytes() +} + +func TbsVerify(our *NodeOur, their *Node, pktEnc *PktEnc) ([]byte, bool, error) { + tbs := TbsPrepare(our, their, pktEnc) + return tbs, ed25519.Verify(their.SignPub, tbs, pktEnc.Sign[:]), nil } func PktEncRead( @@ -293,11 +298,13 @@ func PktEncRead( nodes map[NodeId]*Node, data io.Reader, out io.Writer, -) (*Node, int64, error) { + signatureVerify bool, + sharedKeyCached []byte, +) ([]byte, *Node, int64, error) { var pktEnc PktEnc _, err := xdr.Unmarshal(data, &pktEnc) if err != nil { - return nil, 0, err + return nil, nil, 0, err } switch pktEnc.Magic { case MagicNNCPEv1.B: @@ -313,51 +320,62 @@ func PktEncRead( err = BadMagic } if err != nil { - return nil, 0, err - } - their, known := nodes[*pktEnc.Sender] - if !known { - return nil, 0, errors.New("Unknown sender") + return nil, nil, 0, err } if *pktEnc.Recipient != *our.Id { - return nil, 0, errors.New("Invalid recipient") - } - tbsRaw, verified, err := TbsVerify(our, their, &pktEnc) - if err != nil { - return nil, 0, err - } - if !verified { - return their, 0, errors.New("Invalid signature") + return nil, nil, 0, errors.New("Invalid recipient") + } + var tbsRaw []byte + var their *Node + if signatureVerify { + their = nodes[*pktEnc.Sender] + if their == nil { + return nil, nil, 0, errors.New("Unknown sender") + } + var verified bool + tbsRaw, verified, err = TbsVerify(our, their, &pktEnc) + if err != nil { + return nil, nil, 0, err + } + if !verified { + return nil, their, 0, errors.New("Invalid signature") + } + } else { + tbsRaw = TbsPrepare(our, &Node{Id: pktEnc.Sender}, &pktEnc) } ad := blake3.Sum256(tbsRaw) sharedKey := new([32]byte) - curve25519.ScalarMult(sharedKey, our.ExchPrv, &pktEnc.ExchPub) + if sharedKeyCached == nil { + curve25519.ScalarMult(sharedKey, our.ExchPrv, &pktEnc.ExchPub) + } else { + copy(sharedKey[:], sharedKeyCached) + } key := make([]byte, chacha20poly1305.KeySize) blake3.DeriveKey(key, string(MagicNNCPEv5.B[:]), sharedKey[:]) aead, err := chacha20poly1305.New(key) if err != nil { - return their, 0, err + return sharedKey[:], their, 0, err } nonce := make([]byte, aead.NonceSize()) sizeBuf := make([]byte, 8+aead.Overhead()) if _, err = io.ReadFull(data, sizeBuf); err != nil { - return their, 0, err + return sharedKey[:], their, 0, err } sizeBuf, err = aead.Open(sizeBuf[:0], nonce, sizeBuf, ad[:]) if err != nil { - return their, 0, err + return sharedKey[:], their, 0, err } size := int64(binary.BigEndian.Uint64(sizeBuf)) lr := io.LimitedReader{R: data, N: size} written, err := aeadProcess(aead, nonce, ad[:], false, &lr, out) if err != nil { - return their, int64(written), err + return sharedKey[:], their, int64(written), err } if written != int(size) { - return their, int64(written), io.ErrUnexpectedEOF + return sharedKey[:], their, int64(written), io.ErrUnexpectedEOF } - return their, size, nil + return sharedKey[:], their, size, nil } diff --git a/src/pkt_test.go b/src/pkt_test.go index 079acbd..62efa71 100644 --- a/src/pkt_test.go +++ b/src/pkt_test.go @@ -112,7 +112,7 @@ func TestPktEncRead(t *testing.T) { var pt bytes.Buffer nodes := make(map[NodeId]*Node) nodes[*node1.Id] = node1.Their() - node, sizeGot, err := PktEncRead(node2, nodes, &ct, &pt) + _, node, sizeGot, err := PktEncRead(node2, nodes, &ct, &pt, true, nil) if err != nil { return false } diff --git a/src/sp.go b/src/sp.go index 7e74823..a1f51eb 100644 --- a/src/sp.go +++ b/src/sp.go @@ -658,6 +658,17 @@ func (state *SPState) StartWorkers( } if !state.NoCK { spCheckerOnce.Do(func() { go SPChecker(state.Ctx) }) + go func() { + for job := range state.Ctx.JobsNoCK(state.Node.Id) { + if job.PktEnc.Nice <= state.Nice { + spCheckerTasks <- SPCheckerTask{ + nodeId: state.Node.Id, + hsh: job.HshValue, + done: state.payloads, + } + } + } + }() } // Remaining handshake payload sending diff --git a/src/tmp.go b/src/tmp.go index 88a7ec3..a1ef2e8 100644 --- a/src/tmp.go +++ b/src/tmp.go @@ -37,8 +37,7 @@ func TempFile(dir, prefix string) (*os.File, error) { func (ctx *Ctx) NewTmpFile() (*os.File, error) { jobsPath := filepath.Join(ctx.Spool, "tmp") - var err error - if err = os.MkdirAll(jobsPath, os.FileMode(0777)); err != nil { + if err := os.MkdirAll(jobsPath, os.FileMode(0777)); err != nil { return nil, err } fd, err := TempFile(jobsPath, "") diff --git a/src/toss.go b/src/toss.go index 7537d69..6f91b43 100644 --- a/src/toss.go +++ b/src/toss.go @@ -38,6 +38,7 @@ import ( xdr "github.com/davecgh/go-xdr/xdr2" "github.com/dustin/go-humanize" "github.com/klauspost/compress/zstd" + "golang.org/x/crypto/blake2b" "golang.org/x/crypto/poly1305" ) @@ -64,609 +65,916 @@ func newNotification(fromTo *FromToJSON, subject string, body []byte) io.Reader return strings.NewReader(strings.Join(lines, "\n")) } -func (ctx *Ctx) Toss( - nodeId *NodeId, - nice uint8, - dryRun, doSeen, noFile, noFreq, noExec, noTrns bool, -) bool { - dirLock, err := ctx.LockDir(nodeId, "toss") - if err != nil { - return false +func pktSizeWithoutEnc(pktSize int64) int64 { + pktSize = pktSize - PktEncOverhead - PktOverhead - PktSizeOverhead + pktSizeBlocks := pktSize / (EncBlkSize + poly1305.TagSize) + if pktSize%(EncBlkSize+poly1305.TagSize) != 0 { + pktSize -= poly1305.TagSize } - defer ctx.UnlockDir(dirLock) - isBad := false + pktSize -= pktSizeBlocks * poly1305.TagSize + return pktSize +} + +var JobRepeatProcess = errors.New("needs processing repeat") + +func jobProcess( + ctx *Ctx, + pipeR *io.PipeReader, + pktName string, + les LEs, + sender *Node, + nice uint8, + pktSize uint64, + jobPath string, + decompressor *zstd.Decoder, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea bool, +) error { + defer pipeR.Close() sendmail := ctx.Neigh[*ctx.SelfId].Exec["sendmail"] - decompressor, err := zstd.NewReader(nil) + var pkt Pkt + _, err := xdr.Unmarshal(pipeR, &pkt) if err != nil { - panic(err) + ctx.LogE("rx-unmarshal", les, err, func(les LEs) string { + return fmt.Sprintf("Tossing %s/%s: unmarshal", sender.Name, pktName) + }) + return err } - defer decompressor.Close() - for job := range ctx.Jobs(nodeId, TRx) { - pktName := filepath.Base(job.Path) - les := LEs{ - {"Node", job.PktEnc.Sender}, - {"Pkt", pktName}, - {"Nice", int(job.PktEnc.Nice)}, - } - if job.PktEnc.Nice > nice { - ctx.LogD("rx-too-nice", les, func(les LEs) string { - return fmt.Sprintf( - "Tossing %s/%s: too nice: %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - NicenessFmt(job.PktEnc.Nice), - ) - }) - continue + les = append(les, LE{"Size", int64(pktSize)}) + ctx.LogD("rx", les, func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s (%s)", + sender.Name, pktName, + humanize.IBytes(pktSize), + ) + }) + switch pkt.Type { + case PktTypeExec, PktTypeExecFat: + if noExec { + return nil } - fd, err := os.Open(job.Path) - if err != nil { - ctx.LogE("rx-open", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing %s/%s: opening %s", - ctx.NodeName(job.PktEnc.Sender), pktName, job.Path, - ) - }) - isBad = true - continue + path := bytes.Split(pkt.Path[:int(pkt.PathLen)], []byte{0}) + handle := string(path[0]) + args := make([]string, 0, len(path)-1) + for _, p := range path[1:] { + args = append(args, string(p)) } - - pipeR, pipeW := io.Pipe() - go func(job Job) error { - pipeWB := bufio.NewWriter(pipeW) - _, _, err := PktEncRead(ctx.Self, ctx.Neigh, bufio.NewReader(fd), pipeWB) - fd.Close() // #nosec G104 - if err != nil { - return pipeW.CloseWithError(err) - } - if err = pipeWB.Flush(); err != nil { - return pipeW.CloseWithError(err) - } - return pipeW.Close() - }(job) - var pkt Pkt - var pktSize int64 - var pktSizeBlocks int64 - if _, err = xdr.Unmarshal(pipeR, &pkt); err != nil { - ctx.LogE("rx-unmarshal", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing %s/%s: unmarshal", - ctx.NodeName(job.PktEnc.Sender), pktName, - ) - }) - isBad = true - goto Closing + argsStr := strings.Join(append([]string{handle}, args...), " ") + les = append(les, LE{"Type", "exec"}, LE{"Dst", argsStr}) + cmdline, exists := sender.Exec[handle] + if !exists || len(cmdline) == 0 { + err = errors.New("No handle found") + ctx.LogE( + "rx-no-handle", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing exec %s/%s (%s): %s", + sender.Name, pktName, + humanize.IBytes(pktSize), argsStr, + ) + }, + ) + return err } - pktSize = job.Size - PktEncOverhead - PktOverhead - PktSizeOverhead - pktSizeBlocks = pktSize / (EncBlkSize + poly1305.TagSize) - if pktSize%(EncBlkSize+poly1305.TagSize) != 0 { - pktSize -= poly1305.TagSize + if pkt.Type == PktTypeExec { + if err = decompressor.Reset(pipeR); err != nil { + log.Fatalln(err) + } } - pktSize -= pktSizeBlocks * poly1305.TagSize - les = append(les, LE{"Size", pktSize}) - ctx.LogD("rx", les, func(les LEs) string { - return fmt.Sprintf( - "Tossing %s/%s (%s)", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), + if !dryRun { + cmd := exec.Command(cmdline[0], append(cmdline[1:], args...)...) + cmd.Env = append( + cmd.Env, + "NNCP_SELF="+ctx.Self.Id.String(), + "NNCP_SENDER="+sender.Id.String(), + "NNCP_NICE="+strconv.Itoa(int(pkt.Nice)), ) - }) - - switch pkt.Type { - case PktTypeExec, PktTypeExecFat: - if noExec { - goto Closing - } - path := bytes.Split(pkt.Path[:int(pkt.PathLen)], []byte{0}) - handle := string(path[0]) - args := make([]string, 0, len(path)-1) - for _, p := range path[1:] { - args = append(args, string(p)) - } - argsStr := strings.Join(append([]string{handle}, args...), " ") - les = append(les, LE{"Type", "exec"}, LE{"Dst", argsStr}) - sender := ctx.Neigh[*job.PktEnc.Sender] - cmdline, exists := sender.Exec[handle] - if !exists || len(cmdline) == 0 { - ctx.LogE( - "rx-no-handle", les, errors.New("No handle found"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing exec %s/%s (%s): %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), argsStr, - ) - }, - ) - isBad = true - goto Closing - } if pkt.Type == PktTypeExec { - if err = decompressor.Reset(pipeR); err != nil { - log.Fatalln(err) - } + cmd.Stdin = decompressor + } else { + cmd.Stdin = pipeR } - if !dryRun { - cmd := exec.Command(cmdline[0], append(cmdline[1:], args...)...) - cmd.Env = append( - cmd.Env, - "NNCP_SELF="+ctx.Self.Id.String(), - "NNCP_SENDER="+sender.Id.String(), - "NNCP_NICE="+strconv.Itoa(int(pkt.Nice)), - ) - if pkt.Type == PktTypeExec { - cmd.Stdin = decompressor - } else { - cmd.Stdin = pipeR - } - output, err := cmd.Output() - if err != nil { - ctx.LogE("rx-hande", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing exec %s/%s (%s): %s: handling", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), argsStr, - ) - }) - isBad = true - goto Closing + output, err := cmd.Output() + if err != nil { + ctx.LogE("rx-hande", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing exec %s/%s (%s): %s: handling", + sender.Name, pktName, + humanize.IBytes(uint64(pktSize)), argsStr, + ) + }) + return err + } + if len(sendmail) > 0 && ctx.NotifyExec != nil { + notify, exists := ctx.NotifyExec[sender.Name+"."+handle] + if !exists { + notify, exists = ctx.NotifyExec["*."+handle] } - if len(sendmail) > 0 && ctx.NotifyExec != nil { - notify, exists := ctx.NotifyExec[sender.Name+"."+handle] - if !exists { - notify, exists = ctx.NotifyExec["*."+handle] - } - if exists { - cmd := exec.Command( - sendmail[0], - append(sendmail[1:], notify.To)..., - ) - cmd.Stdin = newNotification(notify, fmt.Sprintf( - "Exec from %s: %s", sender.Name, argsStr, - ), output) - if err = cmd.Run(); err != nil { - ctx.LogE("rx-notify", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing exec %s/%s (%s): %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), argsStr, - ) - }) - } + if exists { + cmd := exec.Command( + sendmail[0], + append(sendmail[1:], notify.To)..., + ) + cmd.Stdin = newNotification(notify, fmt.Sprintf( + "Exec from %s: %s", sender.Name, argsStr, + ), output) + if err = cmd.Run(); err != nil { + ctx.LogE("rx-notify", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing exec %s/%s (%s): %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), argsStr, + ) + }) } } } - ctx.LogI("rx", les, func(les LEs) string { - return fmt.Sprintf( - "Got exec from %s to %s (%s)", - ctx.NodeName(job.PktEnc.Sender), argsStr, - humanize.IBytes(uint64(pktSize)), - ) - }) - if !dryRun { - if doSeen { - if fd, err := os.Create(job.Path + SeenSuffix); err == nil { - fd.Close() // #nosec G104 + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf( + "Got exec from %s to %s (%s)", + sender.Name, argsStr, + humanize.IBytes(pktSize), + ) + }) + if !dryRun && jobPath != "" { + if doSeen { + if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + fd.Close() + if err = DirSync(filepath.Base(jobPath)); err != nil { + ctx.LogE("rx-dirsync", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: dirsyncing", + sender.Name, pktName, + humanize.IBytes(pktSize), + filepath.Base(jobPath), + ) + }) + return err } } - if err = os.Remove(job.Path); err != nil { - ctx.LogE("rx-notify", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing exec %s/%s (%s): %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), argsStr, - ) - }) - isBad = true - } else if ctx.HdrUsage { - os.Remove(job.Path + HdrSuffix) - } } + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx-notify", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing exec %s/%s (%s): %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), argsStr, + ) + }) + return err + } else if ctx.HdrUsage { + os.Remove(jobPath + HdrSuffix) + } + } - case PktTypeFile: - if noFile { - goto Closing - } - dst := string(pkt.Path[:int(pkt.PathLen)]) - les = append(les, LE{"Type", "file"}, LE{"Dst", dst}) - if filepath.IsAbs(dst) { - ctx.LogE( - "rx-non-rel", les, errors.New("non-relative destination path"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }, + case PktTypeFile: + if noFile { + return nil + } + dst := string(pkt.Path[:int(pkt.PathLen)]) + les = append(les, LE{"Type", "file"}, LE{"Dst", dst}) + if filepath.IsAbs(dst) { + err = errors.New("non-relative destination path") + ctx.LogE( + "rx-non-rel", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }, + ) + return err + } + incoming := sender.Incoming + if incoming == nil { + err = errors.New("incoming is not allowed") + ctx.LogE( + "rx-no-incoming", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }, + ) + return err + } + dir := filepath.Join(*incoming, path.Dir(dst)) + if err = os.MkdirAll(dir, os.FileMode(0777)); err != nil { + ctx.LogE("rx-mkdir", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: mkdir", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) - isBad = true - goto Closing + }) + return err + } + if !dryRun { + tmp, err := TempFile(dir, "file") + if err != nil { + ctx.LogE("rx-mktemp", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: mktemp", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err } - incoming := ctx.Neigh[*job.PktEnc.Sender].Incoming - if incoming == nil { - ctx.LogE( - "rx-no-incoming", les, errors.New("incoming is not allowed"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }, + les = append(les, LE{"Tmp", tmp.Name()}) + ctx.LogD("rx-tmp-created", les, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: created: %s", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, tmp.Name(), ) - isBad = true - goto Closing + }) + bufW := bufio.NewWriter(tmp) + if _, err = CopyProgressed( + bufW, pipeR, "Rx file", + append(les, LE{"FullSize", int64(pktSize)}), + ctx.ShowPrgrs, + ); err != nil { + ctx.LogE("rx-copy", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: copying", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err } - dir := filepath.Join(*incoming, path.Dir(dst)) - if err = os.MkdirAll(dir, os.FileMode(0777)); err != nil { - ctx.LogE("rx-mkdir", les, err, func(les LEs) string { + if err = bufW.Flush(); err != nil { + tmp.Close() // #nosec G104 + ctx.LogE("rx-flush", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: mkdir", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: flushing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) }) - isBad = true - goto Closing + return err } - if !dryRun { - tmp, err := TempFile(dir, "file") - if err != nil { - ctx.LogE("rx-mktemp", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: mktemp", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - isBad = true - goto Closing - } - les = append(les, LE{"Tmp", tmp.Name()}) - ctx.LogD("rx-tmp-created", les, func(les LEs) string { + if err = tmp.Sync(); err != nil { + tmp.Close() // #nosec G104 + ctx.LogE("rx-sync", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: created: %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, tmp.Name(), + "Tossing file %s/%s (%s): %s: syncing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) }) - bufW := bufio.NewWriter(tmp) - if _, err = CopyProgressed( - bufW, pipeR, "Rx file", - append(les, LE{"FullSize", pktSize}), - ctx.ShowPrgrs, - ); err != nil { - ctx.LogE("rx-copy", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: copying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - isBad = true - goto Closing - } - if err = bufW.Flush(); err != nil { - tmp.Close() // #nosec G104 - ctx.LogE("rx-flush", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: flushing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - isBad = true - goto Closing - } - if err = tmp.Sync(); err != nil { - tmp.Close() // #nosec G104 - ctx.LogE("rx-sync", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: syncing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, - ) - }) - isBad = true - goto Closing - } - if err = tmp.Close(); err != nil { - ctx.LogE("rx-close", les, err, func(les LEs) string { + return err + } + if err = tmp.Close(); err != nil { + ctx.LogE("rx-close", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: closing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err + } + dstPathOrig := filepath.Join(*incoming, dst) + dstPath := dstPathOrig + dstPathCtr := 0 + for { + if _, err = os.Stat(dstPath); err != nil { + if os.IsNotExist(err) { + break + } + ctx.LogE("rx-stat", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: closing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: stating: %s", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, dstPath, ) }) - isBad = true - goto Closing + return err } - dstPathOrig := filepath.Join(*incoming, dst) - dstPath := dstPathOrig - dstPathCtr := 0 - for { - if _, err = os.Stat(dstPath); err != nil { - if os.IsNotExist(err) { - break + dstPath = dstPathOrig + "." + strconv.Itoa(dstPathCtr) + dstPathCtr++ + } + if err = os.Rename(tmp.Name(), dstPath); err != nil { + ctx.LogE("rx-rename", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: renaming", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err + } + if err = DirSync(*incoming); err != nil { + ctx.LogE("rx-dirsync", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: dirsyncing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err + } + les = les[:len(les)-1] // delete Tmp + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf( + "Got file %s (%s) from %s", + dst, humanize.IBytes(pktSize), sender.Name, + ) + }) + if !dryRun { + if jobPath != "" { + if doSeen { + if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + fd.Close() + if err = DirSync(filepath.Base(jobPath)); err != nil { + ctx.LogE("rx-dirsync", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: dirsyncing", + sender.Name, pktName, + humanize.IBytes(pktSize), + filepath.Base(jobPath), + ) + }) + return err } - ctx.LogE("rx-stat", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: stating: %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, dstPath, - ) - }) - isBad = true - goto Closing } - dstPath = dstPathOrig + "." + strconv.Itoa(dstPathCtr) - dstPathCtr++ } - if err = os.Rename(tmp.Name(), dstPath); err != nil { - ctx.LogE("rx-rename", les, err, func(les LEs) string { + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx-remove", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: renaming", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) }) - isBad = true + return err + } else if ctx.HdrUsage { + os.Remove(jobPath + HdrSuffix) } - if err = DirSync(*incoming); err != nil { - ctx.LogE("rx-dirsync", les, err, func(les LEs) string { + } + if len(sendmail) > 0 && ctx.NotifyFile != nil { + cmd := exec.Command( + sendmail[0], + append(sendmail[1:], ctx.NotifyFile.To)..., + ) + cmd.Stdin = newNotification(ctx.NotifyFile, fmt.Sprintf( + "File from %s: %s (%s)", + sender.Name, dst, humanize.IBytes(pktSize), + ), nil) + if err = cmd.Run(); err != nil { + ctx.LogE("rx-notify", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: dirsyncing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, ) }) - isBad = true } - les = les[:len(les)-1] // delete Tmp } - ctx.LogI("rx", les, func(les LEs) string { + } + + case PktTypeFreq: + if noFreq { + return nil + } + src := string(pkt.Path[:int(pkt.PathLen)]) + les := append(les, LE{"Type", "freq"}, LE{"Src", src}) + if filepath.IsAbs(src) { + err = errors.New("non-relative source path") + ctx.LogE( + "rx-non-rel", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing freq %s/%s (%s): %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), src, + ) + }, + ) + return err + } + dstRaw, err := ioutil.ReadAll(pipeR) + if err != nil { + ctx.LogE("rx-read", les, err, func(les LEs) string { return fmt.Sprintf( - "Got file %s (%s) from %s", - dst, humanize.IBytes(uint64(pktSize)), - ctx.NodeName(job.PktEnc.Sender), + "Tossing freq %s/%s (%s): %s: reading", + sender.Name, pktName, + humanize.IBytes(pktSize), src, ) }) - if !dryRun { + return err + } + dst := string(dstRaw) + les = append(les, LE{"Dst", dst}) + freqPath := sender.FreqPath + if freqPath == nil { + err = errors.New("freqing is not allowed") + ctx.LogE( + "rx-no-freq", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing freq %s/%s (%s): %s -> %s", + sender.Name, pktName, + humanize.IBytes(pktSize), src, dst, + ) + }, + ) + return err + } + if !dryRun { + err = ctx.TxFile( + sender, + pkt.Nice, + filepath.Join(*freqPath, src), + dst, + sender.FreqChunked, + sender.FreqMinSize, + sender.FreqMaxSize, + nil, + ) + if err != nil { + ctx.LogE("rx-tx", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing freq %s/%s (%s): %s -> %s: txing", + sender.Name, pktName, + humanize.IBytes(pktSize), src, dst, + ) + }) + return err + } + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf("Got file request %s to %s", src, sender.Name) + }) + if !dryRun { + if jobPath != "" { if doSeen { - if fd, err := os.Create(job.Path + SeenSuffix); err == nil { - fd.Close() // #nosec G104 + if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + fd.Close() + if err = DirSync(filepath.Base(jobPath)); err != nil { + ctx.LogE("rx-dirsync", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: dirsyncing", + sender.Name, pktName, + humanize.IBytes(pktSize), + filepath.Base(jobPath), + ) + }) + return err + } } } - if err = os.Remove(job.Path); err != nil { + if err = os.Remove(jobPath); err != nil { ctx.LogE("rx-remove", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: removing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing freq %s/%s (%s): %s -> %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), src, dst, ) }) - isBad = true + return err } else if ctx.HdrUsage { - os.Remove(job.Path + HdrSuffix) + os.Remove(jobPath + HdrSuffix) } - if len(sendmail) > 0 && ctx.NotifyFile != nil { - cmd := exec.Command( - sendmail[0], - append(sendmail[1:], ctx.NotifyFile.To)..., - ) - cmd.Stdin = newNotification(ctx.NotifyFile, fmt.Sprintf( - "File from %s: %s (%s)", - ctx.Neigh[*job.PktEnc.Sender].Name, - dst, - humanize.IBytes(uint64(pktSize)), - ), nil) - if err = cmd.Run(); err != nil { - ctx.LogE("rx-notify", les, err, func(les LEs) string { + } + if len(sendmail) > 0 && ctx.NotifyFreq != nil { + cmd := exec.Command( + sendmail[0], + append(sendmail[1:], ctx.NotifyFreq.To)..., + ) + cmd.Stdin = newNotification(ctx.NotifyFreq, fmt.Sprintf( + "Freq from %s: %s", sender.Name, src, + ), nil) + if err = cmd.Run(); err != nil { + ctx.LogE("rx-notify", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing freq %s/%s (%s): %s -> %s: notifying", + sender.Name, pktName, + humanize.IBytes(pktSize), src, dst, + ) + }) + } + } + } + + case PktTypeTrns: + if noTrns { + return nil + } + dst := new([MTHSize]byte) + copy(dst[:], pkt.Path[:int(pkt.PathLen)]) + nodeId := NodeId(*dst) + les := append(les, LE{"Type", "trns"}, LE{"Dst", nodeId}) + logMsg := func(les LEs) string { + return fmt.Sprintf( + "Tossing trns %s/%s (%s): %s", + sender.Name, pktName, + humanize.IBytes(pktSize), + nodeId.String(), + ) + } + node := ctx.Neigh[nodeId] + if node == nil { + err = errors.New("unknown node") + ctx.LogE("rx-unknown", les, err, logMsg) + return err + } + ctx.LogD("rx-tx", les, logMsg) + if !dryRun { + if err = ctx.TxTrns(node, nice, int64(pktSize), pipeR); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { + return logMsg(les) + ": txing" + }) + return err + } + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf( + "Got transitional packet from %s to %s (%s)", + sender.Name, + ctx.NodeName(&nodeId), + humanize.IBytes(pktSize), + ) + }) + if !dryRun && jobPath != "" { + if doSeen { + if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + fd.Close() + if err = DirSync(filepath.Base(jobPath)); err != nil { + ctx.LogE("rx-dirsync", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), dst, + "Tossing file %s/%s (%s): %s: dirsyncing", + sender.Name, pktName, + humanize.IBytes(pktSize), + filepath.Base(jobPath), ) }) + return err } } } - - case PktTypeFreq: - if noFreq { - goto Closing - } - src := string(pkt.Path[:int(pkt.PathLen)]) - les := append(les, LE{"Type", "freq"}, LE{"Src", src}) - if filepath.IsAbs(src) { - ctx.LogE( - "rx-non-rel", les, errors.New("non-relative source path"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, - ) - }, - ) - isBad = true - goto Closing - } - dstRaw, err := ioutil.ReadAll(pipeR) - if err != nil { - ctx.LogE("rx-read", les, err, func(les LEs) string { + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s: reading", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, + "Tossing trns %s/%s (%s): %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), + ctx.NodeName(&nodeId), ) }) - isBad = true - goto Closing - } - dst := string(dstRaw) - les = append(les, LE{"Dst", dst}) - sender := ctx.Neigh[*job.PktEnc.Sender] - freqPath := sender.FreqPath - if freqPath == nil { - ctx.LogE( - "rx-no-freq", les, errors.New("freqing is not allowed"), - func(les LEs) string { - return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s -> %s", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, dst, - ) - }, - ) - isBad = true - goto Closing - } - if !dryRun { - err = ctx.TxFile( - sender, - pkt.Nice, - filepath.Join(*freqPath, src), - dst, - sender.FreqChunked, - sender.FreqMinSize, - sender.FreqMaxSize, + return err + } else if ctx.HdrUsage { + os.Remove(jobPath + HdrSuffix) + } + } + + case PktTypeArea: + if noArea { + return nil + } + areaId := new(AreaId) + copy(areaId[:], pkt.Path[:int(pkt.PathLen)]) + les := append(les, LE{"Type", "area"}, LE{"Area", areaId}) + logMsg := func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s (%s): area %s", + sender.Name, pktName, + humanize.IBytes(pktSize), + ctx.AreaName(areaId), + ) + } + area := ctx.AreaId2Area[*areaId] + if area == nil { + err = errors.New("unknown area") + ctx.LogE("rx-area-unknown", les, err, logMsg) + return err + } + pktEnc, pktEncRaw, err := ctx.HdrRead(pipeR) + fullPipeR := io.MultiReader(bytes.NewReader(pktEncRaw), pipeR) + if err != nil { + ctx.LogE("rx-area-pkt-enc-read", les, err, logMsg) + return err + } + msgHashRaw := blake2b.Sum256(pktEncRaw) + msgHash := Base32Codec.EncodeToString(msgHashRaw[:]) + les = append(les, LE{"AreaMsg", msgHash}) + ctx.LogD("rx-area", les, logMsg) + + if dryRun { + for _, nodeId := range area.Subs { + node := ctx.Neigh[*nodeId] + lesEcho := append(les, LE{"Echo", nodeId}) + seenDir := filepath.Join( + ctx.Spool, nodeId.String(), AreaDir, area.Id.String(), ) - if err != nil { - ctx.LogE("rx-tx", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s -> %s: txing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, dst, - ) + seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + logMsgNode := func(les LEs) string { + return fmt.Sprintf( + "%s: echoing to: %s", logMsg(les), node.Name, + ) + } + if _, err := os.Stat(seenPath); err == nil { + ctx.LogD("rx-area-echo-seen", lesEcho, func(les LEs) string { + return logMsgNode(les) + ": already sent" }) - isBad = true - goto Closing + continue } + ctx.LogI("rx-area-echo", lesEcho, logMsgNode) } - ctx.LogI("rx", les, func(les LEs) string { - return fmt.Sprintf( - "Got file request %s to %s", - src, ctx.NodeName(job.PktEnc.Sender), + } else { + for _, nodeId := range area.Subs { + node := ctx.Neigh[*nodeId] + lesEcho := append(les, LE{"Echo", nodeId}) + seenDir := filepath.Join( + ctx.Spool, nodeId.String(), AreaDir, area.Id.String(), ) - }) - if !dryRun { - if doSeen { - if fd, err := os.Create(job.Path + SeenSuffix); err == nil { - fd.Close() // #nosec G104 + seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + logMsgNode := func(les LEs) string { + return fmt.Sprintf("%s: echo to: %s", logMsg(les), node.Name) + } + if _, err := os.Stat(seenPath); err == nil { + ctx.LogD("rx-area-echo-seen", lesEcho, func(les LEs) string { + return logMsgNode(les) + ": already sent" + }) + continue + } + if nodeId != sender.Id { + ctx.LogI("rx-area-echo", lesEcho, logMsgNode) + if _, err = ctx.Tx( + node, &pkt, nice, int64(pktSize), 0, fullPipeR, pktName, nil, + ); err != nil { + ctx.LogE("rx-area", lesEcho, err, logMsgNode) + return err } } - if err = os.Remove(job.Path); err != nil { - ctx.LogE("rx-remove", les, err, func(les LEs) string { + if err = os.MkdirAll(seenDir, os.FileMode(0777)); err != nil { + ctx.LogE("rx-area-mkdir", lesEcho, err, logMsgNode) + return err + } + if fd, err := os.Create(seenPath); err == nil { + fd.Close() + if err = DirSync(seenDir); err != nil { + ctx.LogE("rx-area-dirsync", les, err, logMsgNode) + return err + } + } else { + ctx.LogE("rx-area-touch", lesEcho, err, logMsgNode) + return err + } + return JobRepeatProcess + } + } + + seenDir := filepath.Join( + ctx.Spool, ctx.SelfId.String(), AreaDir, area.Id.String(), + ) + seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + if _, err := os.Stat(seenPath); err == nil { + ctx.LogD("rx-area-seen", les, func(les LEs) string { + return logMsg(les) + ": already seen" + }) + if !dryRun && jobPath != "" { + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx-area-remove", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s -> %s: removing", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, dst, + "Tossing area %s/%s (%s): %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), + msgHash, ) }) - isBad = true + return err } else if ctx.HdrUsage { - os.Remove(job.Path + HdrSuffix) + os.Remove(jobPath + HdrSuffix) } - if len(sendmail) > 0 && ctx.NotifyFreq != nil { - cmd := exec.Command( - sendmail[0], - append(sendmail[1:], ctx.NotifyFreq.To)..., + } + return nil + } + + if area.Prv == nil { + ctx.LogD("rx-area-no-prv", les, func(les LEs) string { + return logMsg(les) + ": no private key for decoding" + }) + } else { + signatureVerify := true + if _, senderKnown := ctx.Neigh[*pktEnc.Sender]; !senderKnown { + if !area.AllowUnknown { + err = errors.New("unknown sender") + ctx.LogE( + "rx-area-unknown", + append(les, LE{"Sender", pktEnc.Sender}), + err, + func(les LEs) string { + return logMsg(les) + ": sender: " + pktEnc.Sender.String() + }, ) - cmd.Stdin = newNotification(ctx.NotifyFreq, fmt.Sprintf( - "Freq from %s: %s", sender.Name, src, - ), nil) - if err = cmd.Run(); err != nil { - ctx.LogE("rx-notify", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing freq %s/%s (%s): %s -> %s: notifying", - ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), src, dst, - ) - }) - } + return err } + signatureVerify = false + } + areaNodeOur := NodeOur{Id: new(NodeId), ExchPrv: new([32]byte)} + copy(areaNodeOur.Id[:], area.Id[:]) + copy(areaNodeOur.ExchPrv[:], area.Prv[:]) + areaNode := Node{ + Id: new(NodeId), + Name: area.Name, + Incoming: area.Incoming, + Exec: area.Exec, } + copy(areaNode.Id[:], area.Id[:]) + pktName := fmt.Sprintf( + "area/%s/%s", + Base32Codec.EncodeToString(areaId[:]), msgHash, + ) - case PktTypeTrns: - if noTrns { - goto Closing - } - dst := new([MTHSize]byte) - copy(dst[:], pkt.Path[:int(pkt.PathLen)]) - nodeId := NodeId(*dst) - node, known := ctx.Neigh[nodeId] - les := append(les, LE{"Type", "trns"}, LE{"Dst", nodeId}) - logMsg := func(les LEs) string { - return fmt.Sprintf( - "Tossing trns %s/%s (%s): %s", - ctx.NodeName(job.PktEnc.Sender), + pipeR, pipeW := io.Pipe() + errs := make(chan error, 1) + go func() { + errs <- jobProcess( + ctx, + pipeR, pktName, - humanize.IBytes(uint64(pktSize)), - nodeId.String(), + les, + &areaNode, + nice, + uint64(pktSizeWithoutEnc(int64(pktSize))), + "", + decompressor, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, ) + }() + _, _, _, err = PktEncRead( + &areaNodeOur, + ctx.Neigh, + fullPipeR, + pipeW, + signatureVerify, + nil, + ) + if err != nil { + pipeW.CloseWithError(err) + go func() { <-errs }() + return err } - if !known { - ctx.LogE("rx-unknown", les, errors.New("unknown node"), logMsg) - isBad = true - goto Closing + pipeW.Close() + if err = <-errs; err != nil { + return err } - ctx.LogD("rx-tx", les, logMsg) - if !dryRun { - if err = ctx.TxTrns(node, job.PktEnc.Nice, pktSize, pipeR); err != nil { - ctx.LogE("rx", les, err, func(les LEs) string { - return logMsg(les) + ": txing" - }) - isBad = true - goto Closing + } + + if !dryRun && jobPath != "" { + if err = os.MkdirAll(seenDir, os.FileMode(0777)); err != nil { + ctx.LogE("rx-area-mkdir", les, err, logMsg) + return err + } + if fd, err := os.Create(seenPath); err == nil { + fd.Close() + if err = DirSync(seenDir); err != nil { + ctx.LogE("rx-area-dirsync", les, err, logMsg) + return err } } - ctx.LogI("rx", les, func(les LEs) string { + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing area %s/%s (%s): %s: removing", + sender.Name, pktName, + humanize.IBytes(pktSize), + msgHash, + ) + }) + return err + } else if ctx.HdrUsage { + os.Remove(jobPath + HdrSuffix) + } + } + + default: + err = errors.New("unknown type") + ctx.LogE( + "rx-type-unknown", les, err, + func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s (%s)", + sender.Name, pktName, humanize.IBytes(pktSize), + ) + }, + ) + return err + } + return nil +} + +func (ctx *Ctx) Toss( + nodeId *NodeId, + xx TRxTx, + nice uint8, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea bool, +) bool { + dirLock, err := ctx.LockDir(nodeId, "toss") + if err != nil { + return false + } + defer ctx.UnlockDir(dirLock) + isBad := false + decompressor, err := zstd.NewReader(nil) + if err != nil { + panic(err) + } + defer decompressor.Close() + for job := range ctx.Jobs(nodeId, xx) { + pktName := filepath.Base(job.Path) + les := LEs{ + {"Node", job.PktEnc.Sender}, + {"Pkt", pktName}, + {"Nice", int(job.PktEnc.Nice)}, + } + if job.PktEnc.Nice > nice { + ctx.LogD("rx-too-nice", les, func(les LEs) string { return fmt.Sprintf( - "Got transitional packet from %s to %s (%s)", - ctx.NodeName(job.PktEnc.Sender), - ctx.NodeName(&nodeId), - humanize.IBytes(uint64(pktSize)), + "Tossing %s/%s: too nice: %s", + ctx.NodeName(job.PktEnc.Sender), pktName, + NicenessFmt(job.PktEnc.Nice), ) }) - if !dryRun { - if doSeen { - if fd, err := os.Create(job.Path + SeenSuffix); err == nil { - fd.Close() // #nosec G104 - } - } - if err = os.Remove(job.Path); err != nil { - ctx.LogE("rx", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing trns %s/%s (%s): %s: removing", - ctx.NodeName(job.PktEnc.Sender), - pktName, - humanize.IBytes(uint64(pktSize)), - ctx.NodeName(&nodeId), - ) - }) - isBad = true - } else if ctx.HdrUsage { - os.Remove(job.Path + HdrSuffix) - } - } + continue + } + fd, err := os.Open(job.Path) + if err != nil { + ctx.LogE("rx-open", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing %s/%s: opening %s", + ctx.NodeName(job.PktEnc.Sender), pktName, job.Path, + ) + }) + isBad = true + continue + } + errs := make(chan error, 1) + var sharedKey []byte + Retry: + pipeR, pipeW := io.Pipe() + go func() { + errs <- jobProcess( + ctx, + pipeR, + pktName, + les, + ctx.Neigh[*job.PktEnc.Sender], + job.PktEnc.Nice, + uint64(pktSizeWithoutEnc(job.Size)), + job.Path, + decompressor, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, + ) + }() + pipeWB := bufio.NewWriter(pipeW) + sharedKey, _, _, err = PktEncRead( + ctx.Self, + ctx.Neigh, + bufio.NewReader(fd), + pipeWB, + sharedKey == nil, + sharedKey, + ) + if err != nil { + pipeW.CloseWithError(err) + } + if err := pipeWB.Flush(); err != nil { + pipeW.CloseWithError(err) + } + pipeW.Close() - default: - ctx.LogE( - "rx-type-unknown", les, errors.New("unknown type"), - func(les LEs) string { + if err != nil { + isBad = true + fd.Close() + go func() { <-errs }() + continue + } + if err = <-errs; err == JobRepeatProcess { + if _, err = fd.Seek(0, io.SeekStart); err != nil { + ctx.LogE("rx-seek", les, err, func(les LEs) string { return fmt.Sprintf( - "Tossing %s/%s (%s)", + "Tossing %s/%s: can not seek", ctx.NodeName(job.PktEnc.Sender), pktName, - humanize.IBytes(uint64(pktSize)), ) - }, - ) + }) + isBad = true + break + } + goto Retry + } else if err != nil { isBad = true } - Closing: - pipeR.Close() // #nosec G104 + fd.Close() } return isBad } @@ -674,7 +982,7 @@ func (ctx *Ctx) Toss( func (ctx *Ctx) AutoToss( nodeId *NodeId, nice uint8, - doSeen, noFile, noFreq, noExec, noTrns bool, + doSeen, noFile, noFreq, noExec, noTrns, noArea bool, ) (chan struct{}, chan bool) { finish := make(chan struct{}) badCode := make(chan bool) @@ -688,7 +996,9 @@ func (ctx *Ctx) AutoToss( default: } time.Sleep(time.Second) - bad = !ctx.Toss(nodeId, nice, false, doSeen, noFile, noFreq, noExec, noTrns) || bad + bad = !ctx.Toss( + nodeId, TRx, nice, false, + doSeen, noFile, noFreq, noExec, noTrns, noArea) || bad } }() return finish, badCode diff --git a/src/toss_test.go b/src/toss_test.go index a66f43b..deb70f1 100644 --- a/src/toss_test.go +++ b/src/toss_test.go @@ -100,6 +100,7 @@ func TestTossExec(t *testing.T) { 1<<15, false, false, + nil, ); err != nil { t.Error(err) return false @@ -112,13 +113,15 @@ func TestTossExec(t *testing.T) { if len(dirFiles(rxPath)) == 0 { continue } - ctx.Toss(ctx.Self.Id, DefaultNiceExec-1, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec-1, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].Exec = make(map[string][]string) ctx.Neigh[*nodeOur.Id].Exec[handle] = []string{"/bin/sh", "-c", "false"} - ctx.Toss(ctx.Self.Id, DefaultNiceExec, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } @@ -130,7 +133,8 @@ func TestTossExec(t *testing.T) { filepath.Join(spool, "mbox"), ), } - ctx.Toss(ctx.Self.Id, DefaultNiceExec, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) != 0 { return false } @@ -204,6 +208,7 @@ func TestTossFile(t *testing.T) { MaxFileSize, 1<<15, MaxFileSize, + nil, ); err != nil { t.Error(err) return false @@ -211,12 +216,14 @@ func TestTossFile(t *testing.T) { } rxPath := filepath.Join(spool, ctx.Self.Id.String(), string(TRx)) os.Rename(filepath.Join(spool, ctx.Self.Id.String(), string(TTx)), rxPath) - ctx.Toss(ctx.Self.Id, DefaultNiceFile, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].Incoming = &incomingPath - ctx.Toss(ctx.Self.Id, DefaultNiceFile, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) != 0 { return false } @@ -281,6 +288,7 @@ func TestTossFileSameName(t *testing.T) { MaxFileSize, 1<<15, MaxFileSize, + nil, ); err != nil { t.Error(err) return false @@ -289,7 +297,8 @@ func TestTossFileSameName(t *testing.T) { rxPath := filepath.Join(spool, ctx.Self.Id.String(), string(TRx)) os.Rename(filepath.Join(spool, ctx.Self.Id.String(), string(TTx)), rxPath) ctx.Neigh[*nodeOur.Id].Incoming = &incomingPath - ctx.Toss(ctx.Self.Id, DefaultNiceFile, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, + false, false, false, false, false, false, false) expected := make(map[string]struct{}) expected["samefile"] = struct{}{} for i := 0; i < files-1; i++ { @@ -360,12 +369,14 @@ func TestTossFreq(t *testing.T) { txPath := filepath.Join(spool, ctx.Self.Id.String(), string(TTx)) os.Rename(txPath, rxPath) os.MkdirAll(txPath, os.FileMode(0700)) - ctx.Toss(ctx.Self.Id, DefaultNiceFreq, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, + false, false, false, false, false, false, false) if len(dirFiles(txPath)) != 0 || len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].FreqPath = &spool - ctx.Toss(ctx.Self.Id, DefaultNiceFreq, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, + false, false, false, false, false, false, false) if len(dirFiles(txPath)) != 0 || len(dirFiles(rxPath)) == 0 { return false } @@ -378,7 +389,8 @@ func TestTossFreq(t *testing.T) { panic(err) } } - ctx.Toss(ctx.Self.Id, DefaultNiceFreq, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, + false, false, false, false, false, false, false) if len(dirFiles(txPath)) == 0 || len(dirFiles(rxPath)) != 0 { return false } @@ -389,7 +401,7 @@ func TestTossFreq(t *testing.T) { t.Error(err) return false } - _, _, err = PktEncRead(ctx.Self, ctx.Neigh, fd, &buf) + _, _, _, err = PktEncRead(ctx.Self, ctx.Neigh, fd, &buf, true, nil) if err != nil { t.Error(err) return false @@ -483,7 +495,8 @@ func TestTossTrns(t *testing.T) { panic(err) } } - ctx.Toss(ctx.Self.Id, 123, false, false, false, false, false, false) + ctx.Toss(ctx.Self.Id, TRx, 123, + false, false, false, false, false, false, false) if len(dirFiles(rxPath)) != 0 { return false } diff --git a/src/tx.go b/src/tx.go index 14f73d2..a3f44fe 100644 --- a/src/tx.go +++ b/src/tx.go @@ -36,6 +36,7 @@ import ( xdr "github.com/davecgh/go-xdr/xdr2" "github.com/dustin/go-humanize" "github.com/klauspost/compress/zstd" + "golang.org/x/crypto/blake2b" "golang.org/x/crypto/chacha20poly1305" ) @@ -53,7 +54,15 @@ func (ctx *Ctx) Tx( size, minSize int64, src io.Reader, pktName string, + areaId *AreaId, ) (*Node, error) { + var area *Area + if areaId != nil { + area = ctx.AreaId2Area[*areaId] + if area.Prv == nil { + return nil, errors.New("unknown area id") + } + } hops := make([]*Node, 0, 1+len(node.Via)) hops = append(hops, node) lastNode := node @@ -62,7 +71,11 @@ func (ctx *Ctx) Tx( hops = append(hops, lastNode) } expectedSize := size - for i := 0; i < len(hops); i++ { + wrappers := len(hops) + if area != nil { + wrappers++ + } + for i := 0; i < wrappers; i++ { expectedSize = PktEncOverhead + PktSizeOverhead + sizeWithTags(PktOverhead+expectedSize) @@ -80,36 +93,92 @@ func (ctx *Ctx) Tx( } errs := make(chan error) + pktEncRaws := make(chan []byte) curSize := size pipeR, pipeW := io.Pipe() - var pktEncRaw []byte - go func(size int64, src io.Reader, dst io.WriteCloser) { - ctx.LogD("tx", LEs{ - {"Node", hops[0].Id}, - {"Nice", int(nice)}, - {"Size", size}, - }, func(les LEs) string { - return fmt.Sprintf( - "Tx packet to %s (%s) nice: %s", - ctx.NodeName(hops[0].Id), - humanize.IBytes(uint64(size)), - NicenessFmt(nice), - ) - }) - pktEncRaw, err = PktEncWrite( - ctx.Self, hops[0], pkt, nice, size, padSize, src, dst, - ) - errs <- err - dst.Close() // #nosec G104 - }(curSize, src, pipeW) - curSize = PktEncOverhead + - PktSizeOverhead + - sizeWithTags(PktOverhead+curSize) + - padSize - var pipeRPrev io.Reader + if area == nil { + go func(size int64, src io.Reader, dst io.WriteCloser) { + ctx.LogD("tx", LEs{ + {"Node", hops[0].Id}, + {"Nice", int(nice)}, + {"Size", size}, + }, func(les LEs) string { + return fmt.Sprintf( + "Tx packet to %s (%s) nice: %s", + ctx.NodeName(hops[0].Id), + humanize.IBytes(uint64(size)), + NicenessFmt(nice), + ) + }) + pktEncRaw, err := PktEncWrite( + ctx.Self, hops[0], pkt, nice, size, padSize, src, dst, + ) + pktEncRaws <- pktEncRaw + errs <- err + dst.Close() // #nosec G104 + }(curSize, src, pipeW) + curSize = PktEncOverhead + PktSizeOverhead + sizeWithTags(PktOverhead+curSize) + curSize += padSize + } else { + go func(size, padSize int64, src io.Reader, dst io.WriteCloser) { + ctx.LogD("tx", LEs{ + {"Area", area.Id}, + {"Nice", int(nice)}, + {"Size", size}, + }, func(les LEs) string { + return fmt.Sprintf( + "Tx area packet to %s (%s) nice: %s", + ctx.AreaName(areaId), + humanize.IBytes(uint64(size)), + NicenessFmt(nice), + ) + }) + areaNode := Node{Id: new(NodeId), ExchPub: new([32]byte)} + copy(areaNode.Id[:], area.Id[:]) + copy(areaNode.ExchPub[:], area.Pub[:]) + pktEncRaw, err := PktEncWrite( + ctx.Self, &areaNode, pkt, nice, size, padSize, src, dst, + ) + pktEncRaws <- pktEncRaw + errs <- err + dst.Close() // #nosec G104 + }(curSize, padSize, src, pipeW) + curSize = PktEncOverhead + PktSizeOverhead + sizeWithTags(PktOverhead+curSize) + curSize += padSize + pipeRPrev = pipeR + pipeR, pipeW = io.Pipe() + go func(size int64, src io.Reader, dst io.WriteCloser) { + pktArea, err := NewPkt(PktTypeArea, 0, area.Id[:]) + if err != nil { + panic(err) + } + ctx.LogD("tx", LEs{ + {"Node", hops[0].Id}, + {"Nice", int(nice)}, + {"Size", size}, + }, func(les LEs) string { + return fmt.Sprintf( + "Tx packet to %s (%s) nice: %s", + ctx.NodeName(hops[0].Id), + humanize.IBytes(uint64(size)), + NicenessFmt(nice), + ) + }) + pktEncRaw, err := PktEncWrite( + ctx.Self, hops[0], pktArea, nice, size, 0, src, dst, + ) + pktEncRaws <- pktEncRaw + errs <- err + dst.Close() // #nosec G104 + }(curSize, pipeRPrev, pipeW) + curSize = PktEncOverhead + PktSizeOverhead + sizeWithTags(PktOverhead+curSize) + } for i := 1; i < len(hops); i++ { - pktTrns, _ := NewPkt(PktTypeTrns, 0, hops[i-1].Id[:]) + pktTrns, err := NewPkt(PktTypeTrns, 0, hops[i-1].Id[:]) + if err != nil { + panic(err) + } pipeRPrev = pipeR pipeR, pipeW = io.Pipe() go func(node *Node, pkt *Pkt, size int64, src io.Reader, dst io.WriteCloser) { @@ -125,7 +194,8 @@ func (ctx *Ctx) Tx( NicenessFmt(nice), ) }) - _, err := PktEncWrite(ctx.Self, node, pkt, nice, size, 0, src, dst) + pktEncRaw, err := PktEncWrite(ctx.Self, node, pkt, nice, size, 0, src, dst) + pktEncRaws <- pktEncRaw errs <- err dst.Close() // #nosec G104 }(hops[i], pktTrns, curSize, pipeRPrev, pipeW) @@ -139,7 +209,15 @@ func (ctx *Ctx) Tx( ) errs <- err }() - for i := 0; i <= len(hops); i++ { + var pktEncRaw []byte + var pktEncMsg []byte + if area != nil { + pktEncMsg = <-pktEncRaws + } + for i := 0; i < len(hops); i++ { + pktEncRaw = <-pktEncRaws + } + for i := 0; i <= wrappers; i++ { err = <-errs if err != nil { tmp.Fd.Close() // #nosec G104 @@ -155,6 +233,43 @@ func (ctx *Ctx) Tx( if ctx.HdrUsage { ctx.HdrWrite(pktEncRaw, filepath.Join(nodePath, string(TTx), tmp.Checksum())) } + if area != nil { + msgHashRaw := blake2b.Sum256(pktEncMsg) + msgHash := Base32Codec.EncodeToString(msgHashRaw[:]) + seenDir := filepath.Join( + ctx.Spool, ctx.SelfId.String(), AreaDir, areaId.String(), + ) + seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + les := LEs{ + {"Node", node.Id}, + {"Nice", int(nice)}, + {"Size", size}, + {"Area", areaId}, + {"AreaMsg", msgHash}, + } + logMsg := func(les LEs) string { + return fmt.Sprintf( + "Tx area packet to %s (%s) nice: %s, area %s: %s", + ctx.NodeName(node.Id), + humanize.IBytes(uint64(size)), + NicenessFmt(nice), + area.Name, + msgHash, + ) + } + if err = os.MkdirAll(seenDir, os.FileMode(0777)); err != nil { + ctx.LogE("tx-mkdir", les, err, logMsg) + return lastNode, err + } + if fd, err := os.Create(seenPath); err == nil { + fd.Close() + if err = DirSync(seenDir); err != nil { + ctx.LogE("tx-dirsync", les, err, logMsg) + return lastNode, err + } + } + ctx.LogI("tx-area", les, logMsg) + } return lastNode, err } @@ -201,6 +316,9 @@ func throughTmpFile(r io.Reader) ( } r, w := io.Pipe() go func() { + for i := 0; i < aead.NonceSize(); i++ { + nonce[i] = 0 + } if _, err := aeadProcess(aead, nonce, nil, false, bufio.NewReader(src), w); err != nil { w.CloseWithError(err) // #nosec G104 } @@ -344,6 +462,7 @@ func (ctx *Ctx) TxFile( srcPath, dstPath string, chunkSize int64, minSize, maxSize int64, + areaId *AreaId, ) error { dstPathSpecified := false if dstPath == "" { @@ -377,7 +496,7 @@ func (ctx *Ctx) TxFile( if err != nil { return err } - _, err = ctx.Tx(node, pkt, nice, fileSize, minSize, reader, dstPath) + _, err = ctx.Tx(node, pkt, nice, fileSize, minSize, reader, dstPath, areaId) les := LEs{ {"Type", "file"}, {"Node", node.Id}, @@ -439,6 +558,7 @@ func (ctx *Ctx) TxFile( minSize, io.TeeReader(reader, hsh), path, + areaId, ) les := LEs{ {"Type", "file"}, @@ -481,7 +601,7 @@ func (ctx *Ctx) TxFile( return err } metaPktSize := int64(metaBuf.Len()) - _, err = ctx.Tx(node, pkt, nice, metaPktSize, minSize, &metaBuf, path) + _, err = ctx.Tx(node, pkt, nice, metaPktSize, minSize, &metaBuf, path, areaId) les := LEs{ {"Type", "file"}, {"Node", node.Id}, @@ -526,7 +646,7 @@ func (ctx *Ctx) TxFreq( } src := strings.NewReader(dstPath) size := int64(src.Len()) - _, err = ctx.Tx(node, pkt, nice, size, minSize, src, srcPath) + _, err = ctx.Tx(node, pkt, nice, size, minSize, src, srcPath, nil) les := LEs{ {"Type", "freq"}, {"Node", node.Id}, @@ -559,6 +679,7 @@ func (ctx *Ctx) TxExec( minSize int64, useTmp bool, noCompress bool, + areaId *AreaId, ) error { path := make([][]byte, 0, 1+len(args)) path = append(path, []byte(handle)) @@ -569,9 +690,9 @@ func (ctx *Ctx) TxExec( if noCompress { pktType = PktTypeExecFat } - pkt, err := NewPkt(pktType, replyNice, bytes.Join(path, []byte{0})) - if err != nil { - return err + pkt, rerr := NewPkt(pktType, replyNice, bytes.Join(path, []byte{0})) + if rerr != nil { + return rerr } var size int64 @@ -592,15 +713,15 @@ func (ctx *Ctx) TxExec( return err } size = int64(compressed.Len()) - _, err = ctx.Tx(node, pkt, nice, size, minSize, &compressed, handle) + _, rerr = ctx.Tx(node, pkt, nice, size, minSize, &compressed, handle, areaId) } if noCompress && !useTmp { var data bytes.Buffer - if _, err = io.Copy(&data, in); err != nil { + if _, err := io.Copy(&data, in); err != nil { return err } size = int64(data.Len()) - _, err = ctx.Tx(node, pkt, nice, size, minSize, &data, handle) + _, rerr = ctx.Tx(node, pkt, nice, size, minSize, &data, handle, areaId) } if !noCompress && useTmp { r, w := io.Pipe() @@ -631,7 +752,7 @@ func (ctx *Ctx) TxExec( return err } size = fileSize - _, err = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle) + _, rerr = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle, areaId) } if noCompress && useTmp { tmpReader, closer, fileSize, err := throughTmpFile(in) @@ -642,7 +763,7 @@ func (ctx *Ctx) TxExec( return err } size = fileSize - _, err = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle) + _, rerr = ctx.Tx(node, pkt, nice, size, minSize, tmpReader, handle, areaId) } dst := strings.Join(append([]string{handle}, args...), " ") @@ -660,12 +781,12 @@ func (ctx *Ctx) TxExec( ctx.NodeName(node.Id), dst, humanize.IBytes(uint64(size)), ) } - if err == nil { + if rerr == nil { ctx.LogI("tx", les, logMsg) } else { - ctx.LogE("tx", les, err, logMsg) + ctx.LogE("tx", les, rerr, logMsg) } - return err + return rerr } func (ctx *Ctx) TxTrns(node *Node, nice uint8, size int64, src io.Reader) error { diff --git a/src/tx_test.go b/src/tx_test.go index 72c6f61..85a00db 100644 --- a/src/tx_test.go +++ b/src/tx_test.go @@ -84,6 +84,7 @@ func TestTx(t *testing.T) { int64(padSize), src, "pktName", + nil, ) if err != nil { return false @@ -110,7 +111,9 @@ func TestTx(t *testing.T) { vias := append(nodeTgt.Via, nodeTgt.Id) for i, hopId := range vias { hopOur := privates[*hopId] - foundNode, _, err := PktEncRead(hopOur, ctx.Neigh, &bufR, &bufW) + _, foundNode, _, err := PktEncRead( + hopOur, ctx.Neigh, &bufR, &bufW, true, nil, + ) if err != nil { return false }