2 NNCP -- Node to Node copy, utilities for store-and-forward data exchange
3 Copyright (C) 2016-2023 Sergey Matveev <stargrave@stargrave.org>
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 // Reassembly chunked file.
37 xdr "github.com/davecgh/go-xdr/xdr2"
38 "github.com/dustin/go-humanize"
39 "go.cypherpunks.ru/nncp/v8"
43 fmt.Fprint(os.Stderr, nncp.UsageHeader())
44 fmt.Fprint(os.Stderr, "nncp-reass -- reassemble chunked files\n\n")
45 fmt.Fprintf(os.Stderr, "Usage: %s [options] [FILE.nncp.meta]\nOptions:\n", os.Args[0])
47 fmt.Fprint(os.Stderr, `
48 Neither FILE, nor -node nor -all can be set simultaneously,
49 but at least one of them must be specified.
53 func process(ctx *nncp.Ctx, path string, keep, dryRun, stdout, dumpMeta bool) bool {
54 fd, err := os.Open(path)
56 log.Fatalln("Can not open file:", err)
59 var metaPkt nncp.ChunkedMeta
60 les := nncp.LEs{{K: "Path", V: path}}
61 logMsg := func(les nncp.LEs) string {
62 return fmt.Sprintf("Reassembling chunked file \"%s\"", path)
64 if _, err = xdr.Unmarshal(fd, &metaPkt); err != nil {
65 ctx.LogE("reass-bad-meta", les, err, func(les nncp.LEs) string {
66 return logMsg(les) + ": bad meta"
71 if metaPkt.Magic == nncp.MagicNNCPMv1.B {
72 ctx.LogE("reass", les, nncp.MagicNNCPMv1.TooOld(), logMsg)
75 if metaPkt.Magic != nncp.MagicNNCPMv2.B {
76 ctx.LogE("reass", les, nncp.BadMagic, logMsg)
80 metaName := filepath.Base(path)
81 if !strings.HasSuffix(metaName, nncp.ChunkedSuffixMeta) {
82 ctx.LogE("reass", les, errors.New("invalid filename suffix"), logMsg)
85 mainName := strings.TrimSuffix(metaName, nncp.ChunkedSuffixMeta)
87 fmt.Printf("Original filename: %s\n", mainName)
89 "File size: %s (%d bytes)\n",
90 humanize.IBytes(metaPkt.FileSize),
94 "Chunk size: %s (%d bytes)\n",
95 humanize.IBytes(metaPkt.ChunkSize),
98 fmt.Printf("Number of chunks: %d\n", len(metaPkt.Checksums))
99 fmt.Println("Checksums:")
100 for chunkNum, checksum := range metaPkt.Checksums {
101 fmt.Printf("\t%d: %s\n", chunkNum, hex.EncodeToString(checksum[:]))
105 mainDir := filepath.Dir(path)
107 chunksPaths := make([]string, 0, len(metaPkt.Checksums))
108 for i := 0; i < len(metaPkt.Checksums); i++ {
109 chunksPaths = append(
111 filepath.Join(mainDir, mainName+nncp.ChunkedSuffixPart+strconv.Itoa(i)),
115 allChunksExist := true
116 for chunkNum, chunkPath := range chunksPaths {
117 fi, err := os.Stat(chunkPath)
118 lesChunk := append(les, nncp.LE{K: "Chunk", V: chunkNum})
119 if err != nil && errors.Is(err, fs.ErrNotExist) {
120 ctx.LogI("reass-chunk-miss", lesChunk, func(les nncp.LEs) string {
121 return fmt.Sprintf("%s: chunk %d missing", logMsg(les), chunkNum)
123 allChunksExist = false
127 if chunkNum+1 == len(chunksPaths) {
128 left := metaPkt.FileSize % metaPkt.ChunkSize
129 badSize = left != 0 && uint64(fi.Size()) != left
131 badSize = uint64(fi.Size()) != metaPkt.ChunkSize
137 errors.New("invalid size"),
138 func(les nncp.LEs) string {
139 return fmt.Sprintf("%s: chunk %d", logMsg(les), chunkNum)
142 allChunksExist = false
150 allChecksumsGood := true
151 for chunkNum, chunkPath := range chunksPaths {
152 fd, err = os.Open(chunkPath)
154 log.Fatalln("Can not open file:", err)
158 log.Fatalln("Can not stat file:", err)
160 hsh = nncp.MTHNew(fi.Size(), 0)
161 if _, err = nncp.CopyProgressed(
162 hsh, bufio.NewReaderSize(fd, nncp.MTHBlockSize), "check",
163 nncp.LEs{{K: "Pkt", V: chunkPath}, {K: "FullSize", V: fi.Size()}},
169 if !bytes.Equal(hsh.Sum(nil), metaPkt.Checksums[chunkNum][:]) {
172 nncp.LEs{{K: "Path", V: path}, {K: "Chunk", V: chunkNum}},
173 errors.New("checksum is bad"),
174 func(les nncp.LEs) string {
175 return fmt.Sprintf("%s: chunk %d", logMsg(les), chunkNum)
178 allChecksumsGood = false
181 if !allChecksumsGood {
185 ctx.LogI("reass", nncp.LEs{{K: "path", V: path}}, logMsg)
193 les = nncp.LEs{{K: "path", V: path}}
195 tmp, err = nncp.TempFile(mainDir, "reass")
199 les = nncp.LEs{{K: "path", V: path}, {K: "Tmp", V: tmp.Name()}}
200 ctx.LogD("reass-tmp-created", les, func(les nncp.LEs) string {
201 return fmt.Sprintf("%s: temporary %s created", logMsg(les), tmp.Name())
205 dstW := bufio.NewWriter(dst)
208 for chunkNum, chunkPath := range chunksPaths {
209 fd, err = os.Open(chunkPath)
211 log.Fatalln("Can not open file:", err)
215 log.Fatalln("Can not stat file:", err)
217 if _, err = nncp.CopyProgressed(
218 dstW, bufio.NewReaderSize(fd, nncp.MTHBlockSize), "reass",
219 nncp.LEs{{K: "Pkt", V: chunkPath}, {K: "FullSize", V: fi.Size()}},
226 if err = os.Remove(chunkPath); err != nil {
229 append(les, nncp.LE{K: "Chunk", V: chunkNum}), err,
230 func(les nncp.LEs) string {
231 return fmt.Sprintf("%s: chunk %d", logMsg(les), chunkNum)
238 if err = dstW.Flush(); err != nil {
239 log.Fatalln("Can not flush:", err)
243 if err = tmp.Sync(); err != nil {
244 log.Fatalln("Can not sync:", err)
247 if err = tmp.Close(); err != nil {
248 log.Fatalln("Can not close:", err)
251 ctx.LogD("reass-written", les, func(les nncp.LEs) string {
252 return logMsg(les) + ": written"
255 if err = os.Remove(path); err != nil {
256 ctx.LogE("reass-removing", les, err, func(les nncp.LEs) string {
257 return logMsg(les) + ": removing"
263 ctx.LogI("reass", nncp.LEs{{K: "Path", V: path}}, func(les nncp.LEs) string {
264 return logMsg(les) + ": done"
269 dstPathOrig := filepath.Join(mainDir, mainName)
270 dstPath := dstPathOrig
273 if _, err = os.Stat(dstPath); err != nil {
274 if errors.Is(err, fs.ErrNotExist) {
279 dstPath = dstPathOrig + "." + strconv.Itoa(dstPathCtr)
282 if err = os.Rename(tmp.Name(), dstPath); err != nil {
285 if err = nncp.DirSync(mainDir); err != nil {
288 ctx.LogI("reass", nncp.LEs{{K: "Path", V: path}}, func(les nncp.LEs) string {
289 return logMsg(les) + ": done"
294 func findMetas(ctx *nncp.Ctx, dirPath string) []string {
295 dir, err := os.Open(dirPath)
296 logMsg := func(les nncp.LEs) string {
297 return "Finding .meta in " + dirPath
300 ctx.LogE("reass", nncp.LEs{{K: "Path", V: dirPath}}, err, logMsg)
304 entries, err := dir.ReadDir(0)
307 ctx.LogE("reass", nncp.LEs{{K: "Path", V: dirPath}}, err, logMsg)
310 metaPaths := make([]string, 0)
311 for _, entry := range entries {
312 if strings.HasSuffix(entry.Name(), nncp.ChunkedSuffixMeta) {
313 metaPaths = append(metaPaths, filepath.Join(dirPath, entry.Name()))
321 cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file")
322 allNodes = flag.Bool("all", false, "Process all found chunked files for all nodes")
323 nodeRaw = flag.String("node", "", "Process all found chunked files for that node")
324 keep = flag.Bool("keep", false, "Do not remove chunks while assembling")
325 dryRun = flag.Bool("dryrun", false, "Do not assemble whole file")
326 dumpMeta = flag.Bool("dump", false, "Print decoded human-readable FILE.nncp.meta")
327 stdout = flag.Bool("stdout", false, "Output reassembled FILE to stdout")
328 spoolPath = flag.String("spool", "", "Override path to spool")
329 logPath = flag.String("log", "", "Override path to logfile")
330 quiet = flag.Bool("quiet", false, "Print only errors")
331 showPrgrs = flag.Bool("progress", false, "Force progress showing")
332 omitPrgrs = flag.Bool("noprogress", false, "Omit progress showing")
333 debug = flag.Bool("debug", false, "Print debug messages")
334 version = flag.Bool("version", false, "Print version information")
335 warranty = flag.Bool("warranty", false, "Print warranty information")
337 log.SetFlags(log.Lshortfile)
341 fmt.Println(nncp.Warranty)
345 fmt.Println(nncp.VersionGet())
349 ctx, err := nncp.CtxFromCmdline(
359 log.Fatalln("Error during initialization:", err)
362 var nodeOnly *nncp.Node
364 nodeOnly, err = ctx.FindNode(*nodeRaw)
366 log.Fatalln("Invalid -node specified:", err)
370 if !(*allNodes || nodeOnly != nil || flag.NArg() > 0) {
374 if flag.NArg() > 0 && (*allNodes || nodeOnly != nil) {
378 if *allNodes && nodeOnly != nil {
386 if process(ctx, flag.Arg(0), *keep, *dryRun, *stdout, *dumpMeta) {
394 seenMetaPaths := make(map[string]struct{})
395 for _, node := range ctx.Neigh {
396 if node.Incoming == nil {
399 for _, metaPath := range findMetas(ctx, *node.Incoming) {
400 if _, seen := seenMetaPaths[metaPath]; seen {
403 if !process(ctx, metaPath, *keep, *dryRun, false, false) {
406 seenMetaPaths[metaPath] = struct{}{}
410 if nodeOnly.Incoming == nil {
411 log.Fatalln("Specified -node does not allow incoming")
413 for _, metaPath := range findMetas(ctx, *nodeOnly.Incoming) {
414 if !process(ctx, metaPath, *keep, *dryRun, false, false) {