From 73cc79fa20ff19d9ef7ac0e360879e70986ccac7 Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Fri, 22 Nov 2019 22:40:40 +0300 Subject: [PATCH] Ability to send directories, creating pax archive --- VERSION | 2 +- doc/bundles.texi | 4 +- doc/cmds.texi | 9 +++ doc/news.ru.texi | 10 +++ doc/news.texi | 10 +++ src/tx.go | 182 +++++++++++++++++++++++++++++++++++++++-------- 6 files changed, 186 insertions(+), 31 deletions(-) diff --git a/VERSION b/VERSION index 0062ac9..831446c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.0 +5.1.0 diff --git a/doc/bundles.texi b/doc/bundles.texi index 8330cdf..21b3104 100644 --- a/doc/bundles.texi +++ b/doc/bundles.texi @@ -48,8 +48,8 @@ can contain mix of various recipients. @end itemize -Technically bundle is valid POSIX.1-2001 (pax) -@url{http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html, tar archive}, +Technically bundle is valid POSIX.1-2001 +@url{https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_01, pax archive} with directory/files hierarchy identical to that is used in @ref{nncp-xfer}: @file{NNCP/RECIPIENT/SENDER/PACKET}. So bundle can also be created by manual tar-ing of @command{nncp-xfer} resulting directory. diff --git a/doc/cmds.texi b/doc/cmds.texi index 4247054..7097e82 100644 --- a/doc/cmds.texi +++ b/doc/cmds.texi @@ -315,6 +315,15 @@ variable. Encryption is performed in AEAD mode with algorithms. Data is splitted on 128 KiB blocks. Each block is encrypted with increasing nonce counter. +If @file{SRC} points to directory, then +@url{https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_01, pax archive} +will be created on the fly with directory contents and destination +filename @file{.tar} appended. It @strong{won't} contain any entities +metainformation, but modification time with the names. UID/GID are set +to zero. Directories have 777 permissions, files have 666, for being +friendly with @command{umask}. Also each entity will have comment like +@verb{|Autogenerated by NNCP version X.Y.Z built with goXXX|}. + If @option{-chunked} is specified, then source file will be split @ref{Chunked, on chunks}. @option{INT} is the desired chunk size in KiBs. This mode is more CPU hungry. Pay attention that chunk is saved in diff --git a/doc/news.ru.texi b/doc/news.ru.texi index d1fe6d3..e85b0b8 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -1,6 +1,16 @@ @node Новости @section Новости +@node Релиз 5.1.0 +@subsection Релиз 5.1.0 +@itemize + +@item +@command{nncp-file} может отправлять директории, автоматически на лету +создавая pax архив. + +@end itemize + @node Релиз 5.0.0 @subsection Релиз 5.0.0 @itemize diff --git a/doc/news.texi b/doc/news.texi index b57c5ea..e29cf17 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -3,6 +3,16 @@ See also this page @ref{Новости, on russian}. +@node Release 5.1.0 +@section Release 5.1.0 +@itemize + +@item +@command{nncp-file} can send directories, automatically creating pax +archive on the fly. + +@end itemize + @node Release 5.0.0 @section Release 5.0.0 @itemize diff --git a/src/tx.go b/src/tx.go index 0227899..88d4e9a 100644 --- a/src/tx.go +++ b/src/tx.go @@ -18,6 +18,7 @@ along with this program. If not, see . package nncp import ( + "archive/tar" "bufio" "bytes" "crypto/rand" @@ -29,6 +30,7 @@ import ( "path/filepath" "strconv" "strings" + "time" xdr "github.com/davecgh/go-xdr/xdr2" "github.com/klauspost/compress/zstd" @@ -36,6 +38,11 @@ import ( "golang.org/x/crypto/chacha20poly1305" ) +const ( + TarBlockSize = 512 + TarExt = ".tar" +) + func (ctx *Ctx) Tx( node *Node, pkt *Pkt, @@ -109,77 +116,190 @@ func (ctx *Ctx) Tx( return lastNode, err } -func prepareTxFile(srcPath string) (io.Reader, *os.File, int64, error) { - var reader io.Reader - var src *os.File - var fileSize int64 - var err error +type DummyCloser struct{} + +func (dc DummyCloser) Close() error { return nil } + +func prepareTxFile(srcPath string) (reader io.Reader, closer io.Closer, fileSize int64, archived bool, rerr error) { if srcPath == "-" { - src, err = ioutil.TempFile("", "nncp-file") + // Read content from stdin, saving to temporary file, encrypting + // on the fly + src, err := ioutil.TempFile("", "nncp-file") if err != nil { - return nil, nil, 0, err + rerr = err + return } os.Remove(src.Name()) tmpW := bufio.NewWriter(src) tmpKey := make([]byte, chacha20poly1305.KeySize) - if _, err = rand.Read(tmpKey[:]); err != nil { - return nil, nil, 0, err + if _, rerr = rand.Read(tmpKey[:]); rerr != nil { + return } aead, err := chacha20poly1305.New(tmpKey) if err != nil { - return nil, nil, 0, err + rerr = err + return } nonce := make([]byte, aead.NonceSize()) written, err := aeadProcess(aead, nonce, true, bufio.NewReader(os.Stdin), tmpW) if err != nil { - return nil, nil, 0, err + rerr = err + return } fileSize = int64(written) if err = tmpW.Flush(); err != nil { - return nil, nil, 0, err + return } src.Seek(0, io.SeekStart) r, w := io.Pipe() go func() { if _, err := aeadProcess(aead, nonce, false, bufio.NewReader(src), w); err != nil { - panic(err) + w.CloseWithError(err) } }() reader = r - } else { - src, err = os.Open(srcPath) - if err != nil { - return nil, nil, 0, err - } - srcStat, err := src.Stat() + closer = src + return + } + + srcStat, err := os.Stat(srcPath) + if err != nil { + rerr = err + return + } + mode := srcStat.Mode() + + if mode.IsRegular() { + // It is regular file, just send it + src, err := os.Open(srcPath) if err != nil { - return nil, nil, 0, err + rerr = err + return } fileSize = srcStat.Size() reader = bufio.NewReader(src) + closer = src + return + } + + if !mode.IsDir() { + rerr = errors.New("unsupported file type") + return + } + + // It is directory, create PAX archive with its contents + archived = true + basePath := filepath.Base(srcPath) + rootPath, err := filepath.Abs(srcPath) + if err != nil { + rerr = err + return + } + type einfo struct { + path string + modTime time.Time + size int64 + } + dirs := make([]einfo, 0, 1<<10) + files := make([]einfo, 0, 1<<10) + rerr = filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + // directory header, PAX record header+contents + fileSize += TarBlockSize + 2*TarBlockSize + dirs = append(dirs, einfo{path: path, modTime: info.ModTime()}) + } else { + // file header, PAX record header+contents, file content + fileSize += TarBlockSize + 2*TarBlockSize + info.Size() + if n := info.Size() % TarBlockSize; n != 0 { + fileSize += TarBlockSize - n // padding + } + files = append(files, einfo{ + path: path, + modTime: info.ModTime(), + size: info.Size(), + }) + } + return nil + }) + if rerr != nil { + return } - return reader, src, fileSize, nil + + r, w := io.Pipe() + reader = r + closer = DummyCloser{} + fileSize += 2 * TarBlockSize // termination block + + go func() { + tarWr := tar.NewWriter(w) + hdr := tar.Header{ + Typeflag: tar.TypeDir, + Mode: 0777, + PAXRecords: map[string]string{ + "comment": "Autogenerated by " + VersionGet(), + }, + Format: tar.FormatPAX, + } + for _, e := range dirs { + hdr.Name = basePath + e.path[len(rootPath):] + hdr.ModTime = e.modTime + if err = tarWr.WriteHeader(&hdr); err != nil { + w.CloseWithError(err) + } + } + hdr.Typeflag = tar.TypeReg + hdr.Mode = 0666 + for _, e := range files { + hdr.Name = basePath + e.path[len(rootPath):] + hdr.ModTime = e.modTime + hdr.Size = e.size + if err = tarWr.WriteHeader(&hdr); err != nil { + w.CloseWithError(err) + } + fd, err := os.Open(e.path) + if err != nil { + w.CloseWithError(err) + } + _, err = io.Copy(tarWr, bufio.NewReader(fd)) + if err != nil { + w.CloseWithError(err) + } + fd.Close() + } + tarWr.Close() + w.Close() + }() + return } func (ctx *Ctx) TxFile(node *Node, nice uint8, srcPath, dstPath string, minSize int64) error { + dstPathSpecified := false if dstPath == "" { if srcPath == "-" { return errors.New("Must provide destination filename") } dstPath = filepath.Base(srcPath) + } else { + dstPathSpecified = true } dstPath = filepath.Clean(dstPath) if filepath.IsAbs(dstPath) { return errors.New("Relative destination path required") } - pkt, err := NewPkt(PktTypeFile, nice, []byte(dstPath)) + reader, closer, fileSize, archived, err := prepareTxFile(srcPath) + if closer != nil { + defer closer.Close() + } if err != nil { return err } - reader, src, fileSize, err := prepareTxFile(srcPath) - if src != nil { - defer src.Close() + if archived && !dstPathSpecified { + dstPath += TarExt } + pkt, err := NewPkt(PktTypeFile, nice, []byte(dstPath)) if err != nil { return err } @@ -208,23 +328,29 @@ func (ctx *Ctx) TxFileChunked( minSize int64, chunkSize int64, ) error { + dstPathSpecified := false if dstPath == "" { if srcPath == "-" { return errors.New("Must provide destination filename") } dstPath = filepath.Base(srcPath) + } else { + dstPathSpecified = true } dstPath = filepath.Clean(dstPath) if filepath.IsAbs(dstPath) { return errors.New("Relative destination path required") } - reader, src, fileSize, err := prepareTxFile(srcPath) - if src != nil { - defer src.Close() + reader, closer, fileSize, archived, err := prepareTxFile(srcPath) + if closer != nil { + defer closer.Close() } if err != nil { return err } + if archived && !dstPathSpecified { + dstPath += TarExt + } if fileSize <= chunkSize { pkt, err := NewPkt(PktTypeFile, nice, []byte(dstPath)) -- 2.44.0