From 712399b4547bd499528a20209d469b2eb1aaff28 Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Thu, 9 Sep 2021 17:35:32 +0300 Subject: [PATCH] Move .seen and .hdr to subdirectories --- doc/admin.texi | 4 +- doc/call.texi | 2 +- doc/cfg/general.texi | 2 +- doc/cmd/nncp-call.texi | 2 +- doc/cmd/nncp-rm.texi | 9 ++-- doc/cmd/nncp-toss.texi | 2 +- doc/makeinfo.rc | 21 +++++++++ doc/multicast.texi | 16 +++---- doc/news.ru.texi | 23 ++++++++++ doc/news.texi | 23 ++++++++++ doc/nncp.html.do | 3 +- doc/nncp.info.do | 23 +--------- doc/sp.texi | 2 +- doc/spool.texi | 14 +++--- src/cmd/nncp-bundle/main.go | 8 ++-- src/cmd/nncp-call/main.go | 2 +- src/cmd/nncp-caller/main.go | 2 +- src/cmd/nncp-cfgnew/main.go | 2 +- src/cmd/nncp-daemon/main.go | 2 +- src/cmd/nncp-rm/main.go | 92 +++++++++++++++++++++++++++++-------- src/cmd/nncp-toss/main.go | 2 +- src/cmd/nncp-xfer/main.go | 6 ++- src/ctx.go | 3 +- src/jobs.go | 16 +++++-- src/sp.go | 7 ++- src/toss.go | 44 ++++++++++++------ src/tx.go | 2 +- 27 files changed, 234 insertions(+), 100 deletions(-) create mode 100644 doc/makeinfo.rc diff --git a/doc/admin.texi b/doc/admin.texi index 86caeb9..11c26ec 100644 --- a/doc/admin.texi +++ b/doc/admin.texi @@ -24,8 +24,8 @@ NNCP uses following files/directories you should be aware of: checksummed. Can be checksummed (with @file{.nock} extension removing) with @command{nncp-check -nock}. - Also it can contain @file{.seen} files, that should be cleaned too - from time to time. + Also it can contain @file{seen/} and @file{hdr/} subdirectories, + that should be cleaned too from time to time. All of that cleaning tasks can be done with @ref{nncp-rm} utility. diff --git a/doc/call.texi b/doc/call.texi index 52572ba..d1e5d99 100644 --- a/doc/call.texi +++ b/doc/call.texi @@ -81,7 +81,7 @@ configuration option when calling. @item autotoss, -doseen, -nofile, -nofreq, -noexec, -notrns Optionally enable auto tossing: run tosser on node's spool every second -during the call. You can control either are @file{.seen} files must be +during the call. You can control either are @file{seen/} files must be created, or skip any kind of packet processing. @item when-tx-exists diff --git a/doc/cfg/general.texi b/doc/cfg/general.texi index 971b585..ae6ca58 100644 --- a/doc/cfg/general.texi +++ b/doc/cfg/general.texi @@ -36,7 +36,7 @@ You can always force its showing with @option{-progress} command line option anyway. @anchor{CfgNoHdr} @item nohdr -@strong{nohdr} option disables @ref{HdrFile, .hdr} files usage. +@strong{nohdr} option disables @ref{HdrFile, @file{hdr/}} files usage. @end table And optional @ref{MCD, MultiCast Discovery} options: diff --git a/doc/cmd/nncp-call.texi b/doc/cmd/nncp-call.texi index 54609fe..7c2f9c8 100644 --- a/doc/cmd/nncp-call.texi +++ b/doc/cmd/nncp-call.texi @@ -61,5 +61,5 @@ node won't be notified that the file is finished. If you run @ref{nncp-check, @command{nncp-check -nock}}, that will checksum files and strip the @file{.nock} extension, then repeated call to remote node will notify about packet's completion. Also it will be notified if -@ref{nncp-toss, tossing} created @file{.seen} file. +@ref{nncp-toss, tossing} created @file{seen/} file. Read @ref{CfgNoCK, more} about @option{-nock} option. diff --git a/doc/cmd/nncp-rm.texi b/doc/cmd/nncp-rm.texi index 8d2db29..f36178a 100644 --- a/doc/cmd/nncp-rm.texi +++ b/doc/cmd/nncp-rm.texi @@ -33,16 +33,15 @@ failing to be processed. @item @option{-part} option deletes @file{.part}ly downloaded files. -@item @option{-seen} option deletes @file{.seen} files. But it does not -apply to @ref{Multicast, multicast areas} @file{.seen} ones! +@item @option{-seen} option deletes @file{seen/} files. But it does not +apply to @ref{Multicast, multicast areas} ones! @item @option{-nock} option deletes non-checksummed (non-verified) @file{.nock} files. -@item @option{-hdr} option deletes cached @file{.hdr} files. +@item @option{-hdr} option deletes cached @file{hdr/} files. -@item @option{-area} option deletes @file{.seen} files in @file{area/} -subdirectories. +@item @option{-area} option deletes seen files in @file{area/} subdirectories. @end itemize diff --git a/doc/cmd/nncp-toss.texi b/doc/cmd/nncp-toss.texi index fe76801..da4b3be 100644 --- a/doc/cmd/nncp-toss.texi +++ b/doc/cmd/nncp-toss.texi @@ -22,7 +22,7 @@ tells what it will do. @option{INT} seconds in an infinite loop. That can be useful when running this command as a daemon. -@option{-seen} option creates empty @file{XXX.seen} file after +@option{-seen} option creates empty @file{seen/XXX} file after successful tossing of @file{XXX} packet. @ref{nncp-xfer}, @ref{nncp-bundle}, @ref{nncp-daemon} and @ref{nncp-call} commands skip inbound packets that has been already seen, processed and tossed. This diff --git a/doc/makeinfo.rc b/doc/makeinfo.rc new file mode 100644 index 0000000..8d60761 --- /dev/null +++ b/doc/makeinfo.rc @@ -0,0 +1,21 @@ +redo-ifchange \ + ../config \ + ../VERSION \ + *.texi \ + cfg/*.texi \ + cmd/*.texi \ + integration/*.texi \ + pedro.txt \ + pkt/*.texi \ + sp.plantuml.txt \ + usecases.ru/*.texi \ + usecases/*.texi +. ../config +${MAKEINFO:-makeinfo} \ + -D "VERSION `cat ../VERSION`" \ + $MAKEINFO_OPTS \ + --set-customization-variable SECTION_NAME_IN_TITLE=1 \ + --set-customization-variable TREE_TRANSFORMATIONS=complete_tree_nodes_menus \ + --set-customization-variable CLOSE_QUOTE_SYMBOL=\" \ + --set-customization-variable OPEN_QUOTE_SYMBOL=\" \ + --output $3 index.texi diff --git a/doc/multicast.texi b/doc/multicast.texi index 15554f2..fbb31ad 100644 --- a/doc/multicast.texi +++ b/doc/multicast.texi @@ -39,12 +39,12 @@ following: @itemize @item check if that message was already seen (sent or received from) before by the destination node: check existence of - @file{SPOOL/NODE/area/AREA/MsgHash.seen} file. Skip that node if + @file{SPOOL/NODE/area/AREA/MsgHash} file. Skip that node if it exists @item if subscriber's node is not the one we received the packet from, then create outgoing encrypted packet to it, with that area packet inside - @item create corresponding @file{MsgHash.seen} file + @item create corresponding @file{MsgHash} file @item "rewind" the outer encrypted file to the beginning and repeat the whole cycle again, while all of subscribers will "seen" that area's message. @@ -55,7 +55,7 @@ following: consumption. @end itemize @item check if we have seen that area's message before by looking at - @file{SPOOL/SELF/area/AREA/MsgHash.seen}. If so, remove the packet, + @file{SPOOL/SELF/area/AREA/MsgHash}. If so, remove the packet, because it is just a ordinary possible duplicate, finish its processing @item check if we have got corresponding area's private key. If no key exists, then remove the packet, finish its processing -- we just @@ -109,15 +109,15 @@ $ nncp-toss -node self @command{nncp-file} creates an encrypted packet with area packet and encrypted packet inside it, with our own @code{self} node as a recipient (in the @file{SPOOL/SELF/tx} directory). It also creates the -@file{SPOOL/SELF/area/AREA/MSGHASH.seen} file. +@file{SPOOL/SELF/area/AREA/MsgHash} file. @item @command{nncp-toss} sees @file{tx/} file and "opens" it, applying the area message tossing procedure as described above. That will create outgoing packets in @file{SPOOL/nodeB/tx} and @file{SPOOL/nodeD/tx} -directories with @file{SPOOL/nodeB/area/AREA/MSGHASH.seen} -@file{SPOOL/nodeD/area/AREA/MSGHASH.seen} files. Because we already have -@file{SPOOL/SELF/area/AREA/MSGHASH.seen}, that packet is removed then. +directories with @file{SPOOL/nodeB/area/AREA/MsgHash} +@file{SPOOL/nodeD/area/AREA/MsgHash} files. Because we already have +@file{SPOOL/SELF/area/AREA/MsgHash}, that packet is removed then. @item When @code{nodeB} receives the encrypted packet, it sees the area one @@ -131,7 +131,7 @@ not read area's message because it lacks the keys. @item @code{nodeD} receives packets from both @code{nodeA} and @code{nodeB}. Only one of them processed, and other is ignored because corresponding -@file{MSGHASH.seen} file will exist. +@file{MsgHash} file will exist. If @code{nodeD} will receive packet from the @code{nodeB} first, it will relay it to the @code{nodeA} also, that will silently remove it when diff --git a/doc/news.ru.texi b/doc/news.ru.texi index 24933c6..210a440 100644 --- a/doc/news.ru.texi +++ b/doc/news.ru.texi @@ -10,6 +10,29 @@ изменениях в spool директориях, для сокращения накладных расходов на их частое чтение. +@item +@file{.seen} и @file{.hdr} файлы находятся в @file{seen/} и @file{hdr/} +поддиректориях теперь, дабы ускорить сканирование spool областей. +Необходима миграция текущих файлов: + +@example +$ find $NNCPSPOOL -type f -name "*.hdr" -exec rm @{@} + + +$ find $NNCPSPOOL -type d -name rx | while read rx ; do + cd $rx + mkdir -p seen + find . -type f -name "*.seen" | while read fn ; do + mv $fn seen/$@{fn%.seen@} + done +done + +$ find $NNCPSPOOL -type d -name area | while read area ; do + find $area -type f -name "*.seen" | while read fn ; do + mv $fn $@{fn%.seen@} + done +done +@end example + @end itemize @node Релиз 7.6.0 diff --git a/doc/news.texi b/doc/news.texi index c5a7124..ad825f1 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -12,6 +12,29 @@ Experimental @code{kqueue} and @code{inotify} based notifications support about spool directory changes, for reducing their often reading overhead. +@item +@file{.seen} and @file{.hdr} files moved to @file{seen/} and @file{hdr/} +subdirectories, for faster scanning of spool directories. +Current files migration required: + +@example +$ find $NNCPSPOOL -type f -name "*.hdr" -exec rm @{@} + + +$ find $NNCPSPOOL -type d -name rx | while read rx ; do + cd $rx + mkdir -p seen + find . -type f -name "*.seen" | while read fn ; do + mv $fn seen/$@{fn%.seen@} + done +done + +$ find $NNCPSPOOL -type d -name area | while read area ; do + find $area -type f -name "*.seen" | while read fn ; do + mv $fn $@{fn%.seen@} + done +done +@end example + @end itemize @node Release 7_6_0 diff --git a/doc/nncp.html.do b/doc/nncp.html.do index bc4a9c2..9aaa184 100644 --- a/doc/nncp.html.do +++ b/doc/nncp.html.do @@ -1,6 +1,7 @@ +redo-ifchange makeinfo.rc rm -fr nncp.html MAKEINFO_OPTS="$MAKEINFO_OPTS --html --css-include style.css" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable FORMAT_MENU=menu" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable SHOW_TITLE=0" MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable DATE_IN_HEADER=1" -MAKEINFO_OPTS="$MAKEINFO_OPTS" . nncp.info.do +MAKEINFO_OPTS="$MAKEINFO_OPTS" . makeinfo.rc diff --git a/doc/nncp.info.do b/doc/nncp.info.do index 8d60761..47f5ee1 100644 --- a/doc/nncp.info.do +++ b/doc/nncp.info.do @@ -1,21 +1,2 @@ -redo-ifchange \ - ../config \ - ../VERSION \ - *.texi \ - cfg/*.texi \ - cmd/*.texi \ - integration/*.texi \ - pedro.txt \ - pkt/*.texi \ - sp.plantuml.txt \ - usecases.ru/*.texi \ - usecases/*.texi -. ../config -${MAKEINFO:-makeinfo} \ - -D "VERSION `cat ../VERSION`" \ - $MAKEINFO_OPTS \ - --set-customization-variable SECTION_NAME_IN_TITLE=1 \ - --set-customization-variable TREE_TRANSFORMATIONS=complete_tree_nodes_menus \ - --set-customization-variable CLOSE_QUOTE_SYMBOL=\" \ - --set-customization-variable OPEN_QUOTE_SYMBOL=\" \ - --output $3 index.texi +redo-ifchange makeinfo.rc +MAKEINFO_OPTS="--no-split" . makeinfo.rc diff --git a/doc/sp.texi b/doc/sp.texi index 8169e99..3944a75 100644 --- a/doc/sp.texi +++ b/doc/sp.texi @@ -182,7 +182,7 @@ payloads, then send all of remaining in the transport stage. Ignore it if it is too nice. @item If already downloaded file exists, then queue @emph{DONE} sending. - @item If @file{.seen} exists, then queue @emph{DONE} sending. + @item If @file{seen/XXX} exists, then queue @emph{DONE} sending. @item If @file{.part} exists, then queue @emph{FREQ} sending with corresponding offset. @end itemize diff --git a/doc/spool.texi b/doc/spool.texi index 14f8ee2..61ec8c3 100644 --- a/doc/spool.texi +++ b/doc/spool.texi @@ -50,9 +50,9 @@ non-checksummed (NoCK) @strong{fully} received file. Its checksum is verified against its filename either by @ref{nncp-check}, or by working online daemons. If it is correct, then its extension is trimmed. -@item LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ.seen +@item seen/LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ @ref{nncp-toss} utility can be invoked with @option{-seen} option, -leading to creation of @file{.seen} files, telling that the file with +leading to creation of @file{seen/} files, telling that the file with specified hash has already been processed before. It could be useful when there are use-cases where multiple ways of packets transfer available and there is possibility of duplicates reception. You have to @@ -60,9 +60,9 @@ manually remove them, when you do not need them (probably because they are expired). @anchor{HdrFile} -@item LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ.hdr +@item hdr/LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ If no @ref{CfgNoHdr, nohdr} option is enabled in configuration file, -then @file{.hdr} files are automatically created for every ordinary +then @file{hdr/} files are automatically created for every ordinary (fully received and checksummed) packet. It literally contains just the header of the corresponding packet. It will be automatically created even during simple @ref{nncp-stat} call. On filesystems with big @@ -71,9 +71,9 @@ directories, because it prevents unnecessary read-amplification. On other filesystems probably it won't help at all, or even harm performance. -There is a hack: you can create more dense @file{.hdr} allocation by -removing all @file{.hdr} files and then running @command{nncp-stat}, -that will recreate them. In many cases many @file{.hdr} files will be +There is a hack: you can create more dense @file{hdr/} allocation by +removing all @file{hdr/} files and then running @command{nncp-stat}, +that will recreate them. In many cases many @file{hdr/} files will be allocated more or less linearly on the disk, decreasing listing time even more. diff --git a/src/cmd/nncp-bundle/main.go b/src/cmd/nncp-bundle/main.go index 9911395..52ce4f6 100644 --- a/src/cmd/nncp-bundle/main.go +++ b/src/cmd/nncp-bundle/main.go @@ -187,7 +187,7 @@ func main() { if err = os.Remove(job.Path); err != nil { log.Fatalln("Error during deletion:", err) } else if ctx.HdrUsage { - os.Remove(job.Path + nncp.HdrSuffix) + os.Remove(nncp.JobPath2Hdr(job.Path)) } } ctx.LogI( @@ -376,7 +376,7 @@ func main() { if !*dryRun { os.Remove(dstPath) if ctx.HdrUsage { - os.Remove(dstPath + nncp.HdrSuffix) + os.Remove(nncp.JobPath2Hdr(dstPath)) } } } else { @@ -416,7 +416,9 @@ func main() { }) continue } - if _, err = os.Stat(dstPath + nncp.SeenSuffix); err == nil || !os.IsNotExist(err) { + if _, err = os.Stat(filepath.Join( + dstDirPath, nncp.SeenDir, pktName, + )); err == nil || !os.IsNotExist(err) { ctx.LogD("bundle-rx-seen", les, func(les nncp.LEs) string { return logMsg(les) + ": packet already seen" }) diff --git a/src/cmd/nncp-call/main.go b/src/cmd/nncp-call/main.go index 7d12066..03a5a1a 100644 --- a/src/cmd/nncp-call/main.go +++ b/src/cmd/nncp-call/main.go @@ -62,7 +62,7 @@ func main() { maxOnlineTimeSec = flag.Uint("maxonlinetime", 0, "Override maxonlinetime option") autoToss = flag.Bool("autotoss", false, "Toss after call is finished") - autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing") autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") diff --git a/src/cmd/nncp-caller/main.go b/src/cmd/nncp-caller/main.go index 177065f..4024b5b 100644 --- a/src/cmd/nncp-caller/main.go +++ b/src/cmd/nncp-caller/main.go @@ -51,7 +51,7 @@ func main() { warranty = flag.Bool("warranty", false, "Print warranty information") autoToss = flag.Bool("autotoss", false, "Toss after call is finished") - autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing") autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") diff --git a/src/cmd/nncp-cfgnew/main.go b/src/cmd/nncp-cfgnew/main.go index 30bdaa9..56e4e4f 100644 --- a/src/cmd/nncp-cfgnew/main.go +++ b/src/cmd/nncp-cfgnew/main.go @@ -179,7 +179,7 @@ func main() { # umask: "022" # Omit progress showing by default # noprogress: true - # Do not use .hdr files + # Do not use hdr/ files # nohdr: true # MultiCast Discovery: diff --git a/src/cmd/nncp-daemon/main.go b/src/cmd/nncp-daemon/main.go index 266fc7a..4e9231d 100644 --- a/src/cmd/nncp-daemon/main.go +++ b/src/cmd/nncp-daemon/main.go @@ -124,7 +124,7 @@ func main() { warranty = flag.Bool("warranty", false, "Print warranty information") autoToss = flag.Bool("autotoss", false, "Toss after call is finished") - autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing") + autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing") autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing") autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing") autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") diff --git a/src/cmd/nncp-rm/main.go b/src/cmd/nncp-rm/main.go index 39cd0c6..b9ca289 100644 --- a/src/cmd/nncp-rm/main.go +++ b/src/cmd/nncp-rm/main.go @@ -54,15 +54,15 @@ func main() { cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") doAll = flag.Bool("all", false, "Apply remove rules to all nodes") doTmp = flag.Bool("tmp", false, "Remove all temporary files") - doHdr = flag.Bool("hdr", false, "Remove all .hdr files") + doHdr = flag.Bool("hdr", false, "Remove all hdr/ files") doLock = flag.Bool("lock", false, "Remove all lock files") nodeRaw = flag.String("node", "", "Node to remove files in") doRx = flag.Bool("rx", false, "Process received packets") doTx = flag.Bool("tx", false, "Process transfered packets") doPart = flag.Bool("part", false, "Remove only .part files") - doSeen = flag.Bool("seen", false, "Remove only .seen files") + doSeen = flag.Bool("seen", false, "Remove only seen/ files") doNoCK = flag.Bool("nock", false, "Remove only .nock files") - doArea = flag.Bool("area", false, "Remove only area/*.seen files") + doArea = flag.Bool("area", false, "Remove only area/* seen files") older = flag.String("older", "", "XXX{smhd}: only older than XXX number of time units") dryRun = flag.Bool("dryrun", false, "Do not actually remove files") pktRaw = flag.String("pkt", "", "Packet to remove") @@ -204,9 +204,7 @@ func main() { }) return nil } - if (*doSeen && strings.HasSuffix(info.Name(), nncp.SeenSuffix)) || - (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || - (*doHdr && strings.HasSuffix(info.Name(), nncp.HdrSuffix)) || + if (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || (*doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix)) { ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) if *dryRun { @@ -233,7 +231,7 @@ func main() { return nil }) } - if *pktRaw != "" || *doRx || *doSeen || *doNoCK || *doHdr || *doPart { + if *pktRaw != "" || *doRx || *doNoCK || *doPart { if err = remove(nncp.TRx); err != nil { log.Fatalln("Can not remove:", err) } @@ -243,6 +241,63 @@ func main() { log.Fatalln("Can not remove:", err) } } + removeSub := func(p string, everything bool) error { + return filepath.Walk( + p, func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if info.IsDir() { + return nil + } + logMsg := func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + } + if everything { + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD( + "rm-skip", nncp.LEs{{K: "File", V: path}}, + func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", path) + }, + ) + } else if !*dryRun { + return os.Remove(path) + } + return nil + }, + ) + } + if *doRx || *doSeen { + if err = removeSub(filepath.Join( + ctx.Spool, node.Id.String(), string(nncp.TRx), nncp.SeenDir, + ), *doSeen); err != nil { + log.Fatalln("Can not remove:", err) + } + } + if *doRx || *doHdr { + if err = removeSub(filepath.Join( + ctx.Spool, node.Id.String(), string(nncp.TRx), nncp.HdrDir, + ), *doHdr); err != nil { + log.Fatalln("Can not remove:", err) + } + } + if *doTx || *doHdr { + if err = removeSub(filepath.Join( + ctx.Spool, node.Id.String(), string(nncp.TTx), nncp.HdrDir, + ), *doHdr); err != nil { + log.Fatalln("Can not remove:", err) + } + } if *doArea { if err = filepath.Walk( filepath.Join(ctx.Spool, node.Id.String(), nncp.AreaDir), @@ -262,20 +317,17 @@ func main() { }) return nil } - if strings.HasSuffix(info.Name(), nncp.SeenSuffix) { - ctx.LogI( - "rm", - nncp.LEs{{K: "File", V: path}}, - func(les nncp.LEs) string { - return fmt.Sprintf("File %s: removed", path) - }, - ) - if *dryRun { - return nil - } - return os.Remove(path) + ctx.LogI( + "rm", + nncp.LEs{{K: "File", V: path}}, + func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + }, + ) + if *dryRun { + return nil } - return nil + return os.Remove(path) }); err != nil { log.Fatalln("Can not remove:", err) } diff --git a/src/cmd/nncp-toss/main.go b/src/cmd/nncp-toss/main.go index 719b8a4..8a3b8c8 100644 --- a/src/cmd/nncp-toss/main.go +++ b/src/cmd/nncp-toss/main.go @@ -42,7 +42,7 @@ func main() { nodeRaw = flag.String("node", "", "Process only that node") niceRaw = flag.String("nice", nncp.NicenessFmt(255), "Minimal required niceness") dryRun = flag.Bool("dryrun", false, "Do not actually write any tossed data") - doSeen = flag.Bool("seen", false, "Create .seen files") + doSeen = flag.Bool("seen", false, "Create seen/ files") cycle = flag.Uint("cycle", 0, "Repeat tossing after N seconds in infinite loop") noFile = flag.Bool("nofile", false, "Do not process \"file\" packets") noFreq = flag.Bool("nofreq", false, "Do not process \"freq\" packets") diff --git a/src/cmd/nncp-xfer/main.go b/src/cmd/nncp-xfer/main.go index fb7a0e0..e565ece 100644 --- a/src/cmd/nncp-xfer/main.go +++ b/src/cmd/nncp-xfer/main.go @@ -423,7 +423,9 @@ Tx: }) continue } - if _, err = os.Stat(filepath.Join(dstPath, pktName+nncp.SeenSuffix)); err == nil || !os.IsNotExist(err) { + if _, err = os.Stat(filepath.Join( + dstPath, nncp.SeenDir, pktName, + )); err == nil || !os.IsNotExist(err) { ctx.LogD("xfer-tx-seen", les, func(les nncp.LEs) string { return logMsg(les) + ": already seen" }) @@ -518,7 +520,7 @@ Tx: }) isBad = true } else if ctx.HdrUsage { - os.Remove(job.Path + nncp.HdrSuffix) + os.Remove(nncp.JobPath2Hdr(job.Path)) } } } diff --git a/src/ctx.go b/src/ctx.go index b1b590b..cfbe1b5 100644 --- a/src/ctx.go +++ b/src/ctx.go @@ -69,7 +69,8 @@ func (ctx *Ctx) FindNode(id string) (*Node, error) { return node, nil } -func ensureDir(p string) error { +func ensureDir(dirs ...string) error { + p := filepath.Join(dirs...) fi, err := os.Stat(p) if err == nil { if fi.IsDir() { diff --git a/src/jobs.go b/src/jobs.go index 6f18f51..7e920fd 100644 --- a/src/jobs.go +++ b/src/jobs.go @@ -35,7 +35,7 @@ const ( TRx TRxTx = "rx" TTx TRxTx = "tx" - HdrSuffix = ".hdr" + HdrDir = "hdr" ) type Job struct { @@ -45,6 +45,10 @@ type Job struct { HshValue *[MTHSize]byte } +func JobPath2Hdr(jobPath string) string { + return filepath.Join(filepath.Dir(jobPath), HdrDir, filepath.Base(jobPath)) +} + func (ctx *Ctx) HdrRead(r io.Reader) (*PktEnc, []byte, error) { var pktEnc PktEnc _, err := xdr.Unmarshal(r, &pktEnc) @@ -80,7 +84,13 @@ func (ctx *Ctx) HdrWrite(pktEncRaw []byte, tgt string) error { os.Remove(tmpHdr.Name()) return err } - if err = os.Rename(tmpHdr.Name(), tgt+HdrSuffix); err != nil { + if err = ensureDir(filepath.Dir(tgt), HdrDir); err != nil { + ctx.LogE("hdr-write-ensure-mkdir", nil, err, func(les LEs) string { + return "Header writing: ensuring directory" + }) + return err + } + if err = os.Rename(tmpHdr.Name(), JobPath2Hdr(tgt)); err != nil { ctx.LogE("hdr-write-rename", nil, err, func(les LEs) string { return "Header writing: renaming" }) @@ -137,7 +147,7 @@ func (ctx *Ctx) jobsFind(nodeId *NodeId, xx TRxTx, nock, part bool) chan Job { if nock || part { fd, err = os.Open(pth) } else { - fd, err = os.Open(pth + HdrSuffix) + fd, err = os.Open(JobPath2Hdr(pth)) if err != nil && os.IsNotExist(err) { hdrExists = false fd, err = os.Open(pth) diff --git a/src/sp.go b/src/sp.go index 1f2e915..8c212b3 100644 --- a/src/sp.go +++ b/src/sp.go @@ -1263,7 +1263,10 @@ func (state *SPState) ProcessSP(payload []byte) ([][]byte, error) { } continue } - if _, err = os.Stat(pktPath + SeenSuffix); err == nil { + if _, err = os.Stat(filepath.Join( + state.Ctx.Spool, state.Node.Id.String(), string(TRx), + SeenDir, Base32Codec.EncodeToString(info.Hash[:]), + )); err == nil { state.Ctx.LogI("sp-info-seen", lesp, func(les LEs) string { return logMsg(les) + ": already seen" }) @@ -1584,7 +1587,7 @@ func (state *SPState) ProcessSP(payload []byte) ([][]byte, error) { return fmt.Sprintf("Packet %s is sent", pktName) }) if state.Ctx.HdrUsage { - os.Remove(pth + HdrSuffix) + os.Remove(JobPath2Hdr(pth)) } } else { state.Ctx.LogE("sp-done", lesp, err, logMsg) diff --git a/src/toss.go b/src/toss.go index 9b8a6d1..4037831 100644 --- a/src/toss.go +++ b/src/toss.go @@ -43,9 +43,13 @@ import ( ) const ( - SeenSuffix = ".seen" + SeenDir = "seen" ) +func jobPath2Seen(jobPath string) string { + return filepath.Join(filepath.Dir(jobPath), SeenDir, filepath.Base(jobPath)) +} + func newNotification(fromTo *FromToJSON, subject string, body []byte) io.Reader { lines := []string{ "From: " + fromTo.From, @@ -198,7 +202,10 @@ func jobProcess( }) if !dryRun && jobPath != "" { if doSeen { - if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { fd.Close() if err = DirSync(filepath.Dir(jobPath)); err != nil { ctx.LogE("rx-dirsync", les, err, func(les LEs) string { @@ -223,7 +230,7 @@ func jobProcess( }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } @@ -391,7 +398,10 @@ func jobProcess( if !dryRun { if jobPath != "" { if doSeen { - if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { fd.Close() if err = DirSync(filepath.Dir(jobPath)); err != nil { ctx.LogE("rx-dirsync", les, err, func(les LEs) string { @@ -416,7 +426,7 @@ func jobProcess( }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } if len(sendmail) > 0 && ctx.NotifyFile != nil { @@ -516,7 +526,10 @@ func jobProcess( if !dryRun { if jobPath != "" { if doSeen { - if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { fd.Close() if err = DirSync(filepath.Dir(jobPath)); err != nil { ctx.LogE("rx-dirsync", les, err, func(les LEs) string { @@ -541,7 +554,7 @@ func jobProcess( }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } if len(sendmail) > 0 && ctx.NotifyFreq != nil { @@ -629,7 +642,10 @@ func jobProcess( }) if !dryRun && jobPath != "" { if doSeen { - if fd, err := os.Create(jobPath + SeenSuffix); err == nil { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { fd.Close() if err = DirSync(filepath.Dir(jobPath)); err != nil { ctx.LogE("rx-dirsync", les, err, func(les LEs) string { @@ -655,7 +671,7 @@ func jobProcess( }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } @@ -698,7 +714,7 @@ func jobProcess( seenDir := filepath.Join( ctx.Spool, nodeId.String(), AreaDir, area.Id.String(), ) - seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + seenPath := filepath.Join(seenDir, msgHash) logMsgNode := func(les LEs) string { return fmt.Sprintf( "%s: echoing to: %s", logMsg(les), node.Name, @@ -719,7 +735,7 @@ func jobProcess( seenDir := filepath.Join( ctx.Spool, nodeId.String(), AreaDir, area.Id.String(), ) - seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + seenPath := filepath.Join(seenDir, msgHash) logMsgNode := func(les LEs) string { return fmt.Sprintf("%s: echo to: %s", logMsg(les), node.Name) } @@ -759,7 +775,7 @@ func jobProcess( seenDir := filepath.Join( ctx.Spool, ctx.SelfId.String(), AreaDir, area.Id.String(), ) - seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + seenPath := filepath.Join(seenDir, msgHash) if _, err := os.Stat(seenPath); err == nil { ctx.LogD("rx-area-seen", les, func(les LEs) string { return logMsg(les) + ": already seen" @@ -776,7 +792,7 @@ func jobProcess( }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } return nil @@ -876,7 +892,7 @@ func jobProcess( }) return err } else if ctx.HdrUsage { - os.Remove(jobPath + HdrSuffix) + os.Remove(JobPath2Hdr(jobPath)) } } diff --git a/src/tx.go b/src/tx.go index 2b166d1..e096ed6 100644 --- a/src/tx.go +++ b/src/tx.go @@ -239,7 +239,7 @@ func (ctx *Ctx) Tx( seenDir := filepath.Join( ctx.Spool, ctx.SelfId.String(), AreaDir, areaId.String(), ) - seenPath := filepath.Join(seenDir, msgHash+SeenSuffix) + seenPath := filepath.Join(seenDir, msgHash) les := LEs{ {"Node", node.Id}, {"Nice", int(nice)}, -- 2.44.0