From 364b3cba688647d50db7b336d9c577e822072e62 Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Fri, 28 Apr 2017 16:18:30 +0300 Subject: [PATCH] Chunked files documentation --- doc/chunked.texi | 42 +++++++++ doc/cmds.texi | 86 ++++++++++++++++--- doc/index.texi | 2 + .../nncp/cmd/nncp-reass/main.go | 4 +- 4 files changed, 120 insertions(+), 14 deletions(-) create mode 100644 doc/chunked.texi diff --git a/doc/chunked.texi b/doc/chunked.texi new file mode 100644 index 0000000..41b1d7a --- /dev/null +++ b/doc/chunked.texi @@ -0,0 +1,42 @@ +@node Chunked +@unnumbered Chunked files + +There is ability to transfer huge files with splitting them into smaller +chunks. Each chunk is treated like a separate file, producing separate +outbound packet unrelated with other ones. + +This is useful when your removable storage device has smaller capacity +than huge file's size. You can transfer those chunks on different +storage devices, and/or at different time, reassembling the whole packet +on the destination node. + +Splitting is done with @ref{nncp-file, nncp-file -chunk} command and +reassembling with @ref{nncp-reass} command. + +Chunked @file{FILE} produces @file{FILE.nncp.meta}, +@file{FILE.nncp.chunk0}, @file{FILE.nncp.chunk1}, ... files. All +@file{.nncp.chunkXXX} can be concatenated together to produce original +@file{FILE}. @file{.nncp.meta} contains information about file/chunk +size and their hash checksums: + +@verbatim ++------------------------------+---------------------+ +| MAGIC | FILESIZE | CHUNKSIZE | HASH0 | HASH1 | ... | ++------------------------------+---------------------+ +@end verbatim + +@multitable @columnfractions 0.2 0.3 0.5 +@headitem @tab XDR type @tab Value +@item Magic number @tab + 8-byte, fixed length opaque data @tab + @verb{|N N C P M 0x00 0x00 0x01|} +@item File size @tab + unsigned hyper integer @tab + Whole reassembled file's size +@item Chunk size @tab + unsigned hyper integer @tab + Size of each chunks (except for the last one, that could be smaller). +@item Checksums @tab + variable length array of 32 byte fixed length opaque data @tab + BLAKE2b-256 checksum of each chunk +@end multitable diff --git a/doc/cmds.texi b/doc/cmds.texi index 70586a7..8cd8e0d 100644 --- a/doc/cmds.texi +++ b/doc/cmds.texi @@ -113,7 +113,7 @@ bind to and listen. @section nncp-file @verbatim -% nncp-file [options] SRC NODE:[DST] +% nncp-file [options] [-chunk INT] SRC NODE:[DST] @end verbatim Send @file{SRC} file to remote @option{NODE}. @file{DST} specifies @@ -131,6 +131,11 @@ packet creation. Pay attention that if you want to send 1 GiB of data taken from stdin, then you have to have 2 GiB of disk space for that temporary file and resulting encrypted packet. +If @option{-chunk} is specified, then source file will be split +@ref{Chunked, on chunks}. @option{INT} is the desired chunk size in +KiBs. This mode is more CPU hungry. Pay attention that chunk is saved in +spool directory immediately and it is not deleted if any error occurs. + If @ref{CfgNotify, notification} is enabled on the remote side for file transmissions, then it will sent simple letter after successful file receiving. @@ -236,6 +241,74 @@ And with the @option{-dump} option it will give you the actual payload tries to zlib-decompress the data from plain packet (useful for mail packets). +@node nncp-reass +@section nncp-reass + +@verbatim +% nncp-reass [options] [-dryrun] [-keep] [-dump] [-stdout] FILE.nncp.meta +% nncp-reass [options] [-dryrun] [-keep] {-all | -node NODE} +@end verbatim + +Reassemble @ref{Chunked, chunked file} after @ref{nncp-toss, tossing}. + +When called with @option{FILE} option, this command will reassemble only +it. When called with @option{-node} option, this command will try to +reassemble all @file{.nncp.meta} files found in @option{NODE}'s +@ref{CfgIncoming, incoming} directory. When called with @option{-all} +option, then cycle through all known nodes to do the same. + +Reassembling process does the following: + +@enumerate +@item Parses @ref{Chunked, @file{.nncp.meta}} file. +@item Checks existence and size of every @file{.nncp.chunkXXX}. +@item Verifies integrity of every chunk. +@item Concatenates all chunks, simultaneously removing them from filesystem. +@end enumerate + +That process reads the whole data twice. Be sure to have free disk +space for at least one chunk. Decrypted chunk files as a rule are saved +in pseudo-random order, so removing them during reassembly process will +likely lead to filesystem fragmentation. Reassembly process on +filesystems with deduplication capability should be rather lightweight. + +If @option{-dryrun} option is specified, then only existence and +integrity checking is performed. + +If @option{-keep} option is specified, then no +@file{.nncp.meta}/@file{.nncp.chunkXXX} files are deleted during +reassembly process. + +@option{-stdout} option outputs reassembled file to stdout, instead of +saving to temporary file with renaming after. This could be useful for +reassembling on separate filesystem to lower fragmentation effect, +and/or separate storage device for higher performance. + +@option{-dump} option prints meta-file contents in human-friendly form. +It is useful mainly for debugging purposes. For example: +@verbatim +Original filename: testfile +File size: 3.8 MiB (3987795 bytes) +Chunk size: 1.0 MiB (1048576 bytes) +Number of chunks: 4 +Checksums: + 0: eac60d819edf40b8ecdacd0b9a5a8c62de2d15eef3c8ca719eafa0be9b894017 + 1: 013a07e659f2e353d0e4339c3375c96c7fffaa2fa00875635f440bbc4631052a + 2: f4f883975a663f2252328707a30e71b2678f933b2f3103db8475b03293e4316e + 3: 0e9e229501bf0ca42d4aa07393d19406d40b179f3922a3986ef12b41019b45a3 +@end verbatim + +@node nncp-rm +@section nncp-rm + +@verbatim +% nncp-rm [options] NODE PKT +@end verbatim + +Remove specified packet (Base32 name) in @option{NODE}'s queues. This +command is useful when you want to remove the packet that is failing to +be processed. + @node nncp-stat @section nncp-stat @@ -295,14 +368,3 @@ configuration file version without any private keys. @file{DIR} directory has the following structure: @file{RECIPIENT/SENDER/PACKET}, where @file{RECIPIENT} is Base32 encoded destination node, @file{SENDER} is Base32 encoded sender node. - -@node nncp-rm -@section nncp-rm - -@verbatim -% nncp-rm [options] NODE PKT -@end verbatim - -Remove specified packet (Base32 name) in @option{NODE}'s queues. This -command is useful when you want to remove the packet that is failing to -be processed. diff --git a/doc/index.texi b/doc/index.texi index ceadf1c..bbc485b 100644 --- a/doc/index.texi +++ b/doc/index.texi @@ -67,6 +67,7 @@ integration with current SMTP servers are the reasons. * Integration with Postfix: Postfix. * Commands:: * Niceness:: +* Chunked files: Chunked. * Spool directory: Spool. * Log format: Log. * Packet format: Packet. @@ -86,6 +87,7 @@ integration with current SMTP servers are the reasons. @include postfix.texi @include cmds.texi @include niceness.texi +@include chunked.texi @include spool.texi @include log.texi @include pkt.texi diff --git a/src/cypherpunks.ru/nncp/cmd/nncp-reass/main.go b/src/cypherpunks.ru/nncp/cmd/nncp-reass/main.go index 1bb6009..45354d7 100644 --- a/src/cypherpunks.ru/nncp/cmd/nncp-reass/main.go +++ b/src/cypherpunks.ru/nncp/cmd/nncp-reass/main.go @@ -43,7 +43,7 @@ import ( func usage() { fmt.Fprintf(os.Stderr, nncp.UsageHeader()) fmt.Fprintln(os.Stderr, "nncp-reass -- reassemble chunked files\n") - fmt.Fprintf(os.Stderr, "Usage: %s [options] [FILE]\nOptions:\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [options] [FILE.nncp.meta]\nOptions:\n", os.Args[0]) flag.PrintDefaults() fmt.Fprint(os.Stderr, ` Neither FILE, nor -node nor -all can be set simultaneously, @@ -269,7 +269,7 @@ func main() { nodeRaw = flag.String("node", "", "Process all found chunked files for that node") keep = flag.Bool("keep", false, "Do not remove chunks while assembling") dryRun = flag.Bool("dryrun", false, "Do not assemble whole file") - dumpMeta = flag.Bool("dump", false, "Print decoded human-readable meta FILE") + dumpMeta = flag.Bool("dump", false, "Print decoded human-readable FILE.nncp.meta") stdout = flag.Bool("stdout", false, "Output reassembled FILE to stdout") quiet = flag.Bool("quiet", false, "Print only errors") debug = flag.Bool("debug", false, "Print debug messages") -- 2.44.0