]> Cypherpunks.ru repositories - nncp.git/commitdiff
Move .seen and .hdr to subdirectories
authorSergey Matveev <stargrave@stargrave.org>
Thu, 9 Sep 2021 14:35:32 +0000 (17:35 +0300)
committerSergey Matveev <stargrave@stargrave.org>
Fri, 10 Sep 2021 12:55:40 +0000 (15:55 +0300)
27 files changed:
doc/admin.texi
doc/call.texi
doc/cfg/general.texi
doc/cmd/nncp-call.texi
doc/cmd/nncp-rm.texi
doc/cmd/nncp-toss.texi
doc/makeinfo.rc [new file with mode: 0644]
doc/multicast.texi
doc/news.ru.texi
doc/news.texi
doc/nncp.html.do
doc/nncp.info.do
doc/sp.texi
doc/spool.texi
src/cmd/nncp-bundle/main.go
src/cmd/nncp-call/main.go
src/cmd/nncp-caller/main.go
src/cmd/nncp-cfgnew/main.go
src/cmd/nncp-daemon/main.go
src/cmd/nncp-rm/main.go
src/cmd/nncp-toss/main.go
src/cmd/nncp-xfer/main.go
src/ctx.go
src/jobs.go
src/sp.go
src/toss.go
src/tx.go

index 86caeb9264f0c76ff5293845f97c51639e7d5966..11c26ecd570b8c42dd0fdd5855eab68c3603bebc 100644 (file)
@@ -24,8 +24,8 @@ NNCP uses following files/directories you should be aware of:
     checksummed. Can be checksummed (with @file{.nock} extension removing)
     with @command{nncp-check -nock}.
 
-    Also it can contain @file{.seen} files, that should be cleaned too
-    from time to time.
+    Also it can contain @file{seen/} and @file{hdr/} subdirectories,
+    that should be cleaned too from time to time.
 
     All of that cleaning tasks can be done with @ref{nncp-rm} utility.
 
index 52572baa11558f2a9a62a94a3d496a14510b5572..d1e5d993489d5f2fcbcd95ffce3eeb900c53283a 100644 (file)
@@ -81,7 +81,7 @@ configuration option when calling.
 
 @item autotoss, -doseen, -nofile, -nofreq, -noexec, -notrns
 Optionally enable auto tossing: run tosser on node's spool every second
-during the call. You can control either are @file{.seen} files must be
+during the call. You can control either are @file{seen/} files must be
 created, or skip any kind of packet processing.
 
 @item when-tx-exists
index 971b585025ead47d384a1ed60f6aff2f270c62e4..ae6ca588827c9b778aeb405c9eb0e60f5a27f59e 100644 (file)
@@ -36,7 +36,7 @@ You can always force its showing with @option{-progress} command line
 option anyway.
 @anchor{CfgNoHdr}
 @item nohdr
-@strong{nohdr} option disables @ref{HdrFile, .hdr} files usage.
+@strong{nohdr} option disables @ref{HdrFile, @file{hdr/}} files usage.
 @end table
 
 And optional @ref{MCD, MultiCast Discovery} options:
index 54609fe489a638733436ff7dccd3da4b07342d03..7c2f9c85308ce8478eaf18d228103db23d14d3c9 100644 (file)
@@ -61,5 +61,5 @@ node won't be notified that the file is finished. If you run
 @ref{nncp-check, @command{nncp-check -nock}}, that will checksum files
 and strip the @file{.nock} extension, then repeated call to remote node
 will notify about packet's completion. Also it will be notified if
-@ref{nncp-toss, tossing} created @file{.seen} file.
+@ref{nncp-toss, tossing} created @file{seen/} file.
 Read @ref{CfgNoCK, more} about @option{-nock} option.
index 8d2db29f26cb009229d81ffd9ed34926c127c266..f36178a045adb19538f3cbc192ff43ee6837181c 100644 (file)
@@ -33,16 +33,15 @@ failing to be processed.
 
 @item @option{-part} option deletes @file{.part}ly downloaded files.
 
-@item @option{-seen} option deletes @file{.seen} files. But it does not
-apply to @ref{Multicast, multicast areas} @file{.seen} ones!
+@item @option{-seen} option deletes @file{seen/} files. But it does not
+apply to @ref{Multicast, multicast areas} ones!
 
 @item @option{-nock} option deletes non-checksummed (non-verified)
 @file{.nock} files.
 
-@item @option{-hdr} option deletes cached @file{.hdr} files.
+@item @option{-hdr} option deletes cached @file{hdr/} files.
 
-@item @option{-area} option deletes @file{.seen} files in @file{area/}
-subdirectories.
+@item @option{-area} option deletes seen files in @file{area/} subdirectories.
 
 @end itemize
 
index fe7680160c521d6bb1405121cea5e7f61b73cdc9..da4b3bef04923757223a67b314a028640d669de1 100644 (file)
@@ -22,7 +22,7 @@ tells what it will do.
 @option{INT} seconds in an infinite loop. That can be useful when
 running this command as a daemon.
 
-@option{-seen} option creates empty @file{XXX.seen} file after
+@option{-seen} option creates empty @file{seen/XXX} file after
 successful tossing of @file{XXX} packet. @ref{nncp-xfer},
 @ref{nncp-bundle}, @ref{nncp-daemon} and @ref{nncp-call} commands skip
 inbound packets that has been already seen, processed and tossed. This
diff --git a/doc/makeinfo.rc b/doc/makeinfo.rc
new file mode 100644 (file)
index 0000000..8d60761
--- /dev/null
@@ -0,0 +1,21 @@
+redo-ifchange \
+    ../config \
+    ../VERSION \
+    *.texi \
+    cfg/*.texi \
+    cmd/*.texi \
+    integration/*.texi \
+    pedro.txt \
+    pkt/*.texi \
+    sp.plantuml.txt \
+    usecases.ru/*.texi \
+    usecases/*.texi
+. ../config
+${MAKEINFO:-makeinfo} \
+    -D "VERSION `cat ../VERSION`" \
+    $MAKEINFO_OPTS \
+    --set-customization-variable SECTION_NAME_IN_TITLE=1 \
+    --set-customization-variable TREE_TRANSFORMATIONS=complete_tree_nodes_menus \
+    --set-customization-variable CLOSE_QUOTE_SYMBOL=\" \
+    --set-customization-variable OPEN_QUOTE_SYMBOL=\" \
+    --output $3 index.texi
index 15554f28451af6cd19c862c35adc505ad96bdf48..fbb31adf94ce0c6444c0d0c85cb06131822a3e17 100644 (file)
@@ -39,12 +39,12 @@ following:
     @itemize
     @item check if that message was already seen (sent or received from)
         before by the destination node: check existence of
-        @file{SPOOL/NODE/area/AREA/MsgHash.seen} file. Skip that node if
+        @file{SPOOL/NODE/area/AREA/MsgHash} file. Skip that node if
         it exists
     @item if subscriber's node is not the one we received the packet
         from, then create outgoing encrypted packet to it, with that
         area packet inside
-    @item create corresponding @file{MsgHash.seen} file
+    @item create corresponding @file{MsgHash} file
     @item "rewind" the outer encrypted file to the beginning and repeat
         the whole cycle again, while all of subscribers will "seen" that
         area's message.
@@ -55,7 +55,7 @@ following:
         consumption.
     @end itemize
 @item check if we have seen that area's message before by looking at
-    @file{SPOOL/SELF/area/AREA/MsgHash.seen}. If so, remove the packet,
+    @file{SPOOL/SELF/area/AREA/MsgHash}. If so, remove the packet,
     because it is just a ordinary possible duplicate, finish its processing
 @item check if we have got corresponding area's private key. If no key
     exists, then remove the packet, finish its processing -- we just
@@ -109,15 +109,15 @@ $ nncp-toss -node self
 @command{nncp-file} creates an encrypted packet with area packet and
 encrypted packet inside it, with our own @code{self} node as a recipient
 (in the @file{SPOOL/SELF/tx} directory). It also creates the
-@file{SPOOL/SELF/area/AREA/MSGHASH.seen} file.
+@file{SPOOL/SELF/area/AREA/MsgHash} file.
 
 @item
 @command{nncp-toss} sees @file{tx/} file and "opens" it, applying the
 area message tossing procedure as described above. That will create
 outgoing packets in @file{SPOOL/nodeB/tx} and @file{SPOOL/nodeD/tx}
-directories with @file{SPOOL/nodeB/area/AREA/MSGHASH.seen}
-@file{SPOOL/nodeD/area/AREA/MSGHASH.seen} files. Because we already have
-@file{SPOOL/SELF/area/AREA/MSGHASH.seen}, that packet is removed then.
+directories with @file{SPOOL/nodeB/area/AREA/MsgHash}
+@file{SPOOL/nodeD/area/AREA/MsgHash} files. Because we already have
+@file{SPOOL/SELF/area/AREA/MsgHash}, that packet is removed then.
 
 @item
 When @code{nodeB} receives the encrypted packet, it sees the area one
@@ -131,7 +131,7 @@ not read area's message because it lacks the keys.
 @item
 @code{nodeD} receives packets from both @code{nodeA} and @code{nodeB}.
 Only one of them processed, and other is ignored because corresponding
-@file{MSGHASH.seen} file will exist.
+@file{MsgHash} file will exist.
 
 If @code{nodeD} will receive packet from the @code{nodeB} first, it will
 relay it to the @code{nodeA} also, that will silently remove it when
index 24933c6a91d7d531fb8e8ac14efc291af8e78f48..210a44016bc343f1c4dc0703dd8591a8452c23c0 100644 (file)
 изменениях в spool директориях, для сокращения накладных расходов на их
 частое чтение.
 
+@item
+@file{.seen} и @file{.hdr} файлы находятся в @file{seen/} и @file{hdr/}
+поддиректориях теперь, дабы ускорить сканирование spool областей.
+Необходима миграция текущих файлов:
+
+@example
+$ find $NNCPSPOOL -type f -name "*.hdr" -exec rm @{@} +
+
+$ find $NNCPSPOOL -type d -name rx | while read rx ; do
+    cd $rx
+    mkdir -p seen
+    find . -type f -name "*.seen" | while read fn ; do
+        mv $fn seen/$@{fn%.seen@}
+    done
+done
+
+$ find $NNCPSPOOL -type d -name area | while read area ; do
+    find $area -type f -name "*.seen" | while read fn ; do
+        mv $fn $@{fn%.seen@}
+    done
+done
+@end example
+
 @end itemize
 
 @node Релиз 7.6.0
index c5a7124ee5843078dcb92229b04c5194855836c7..ad825f10b7be2e4a5e5504f2627f39da1e3be1dc 100644 (file)
@@ -12,6 +12,29 @@ Experimental @code{kqueue} and @code{inotify} based notifications
 support about spool directory changes, for reducing their often reading
 overhead.
 
+@item
+@file{.seen} and @file{.hdr} files moved to @file{seen/} and @file{hdr/}
+subdirectories, for faster scanning of spool directories.
+Current files migration required:
+
+@example
+$ find $NNCPSPOOL -type f -name "*.hdr" -exec rm @{@} +
+
+$ find $NNCPSPOOL -type d -name rx | while read rx ; do
+    cd $rx
+    mkdir -p seen
+    find . -type f -name "*.seen" | while read fn ; do
+        mv $fn seen/$@{fn%.seen@}
+    done
+done
+
+$ find $NNCPSPOOL -type d -name area | while read area ; do
+    find $area -type f -name "*.seen" | while read fn ; do
+        mv $fn $@{fn%.seen@}
+    done
+done
+@end example
+
 @end itemize
 
 @node Release 7_6_0
index bc4a9c23379773e5de6f86c814d8543d0643f66c..9aaa1841ecbba7b59c2a20d72cb65ccef1cfe985 100644 (file)
@@ -1,6 +1,7 @@
+redo-ifchange makeinfo.rc
 rm -fr nncp.html
 MAKEINFO_OPTS="$MAKEINFO_OPTS --html --css-include style.css"
 MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable FORMAT_MENU=menu"
 MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable SHOW_TITLE=0"
 MAKEINFO_OPTS="$MAKEINFO_OPTS --set-customization-variable DATE_IN_HEADER=1"
-MAKEINFO_OPTS="$MAKEINFO_OPTS" . nncp.info.do
+MAKEINFO_OPTS="$MAKEINFO_OPTS" . makeinfo.rc
index 8d607611489b74af21cfa21e031c12c3afca47ed..47f5ee1413d67397c034043957aad74350e812d8 100644 (file)
@@ -1,21 +1,2 @@
-redo-ifchange \
-    ../config \
-    ../VERSION \
-    *.texi \
-    cfg/*.texi \
-    cmd/*.texi \
-    integration/*.texi \
-    pedro.txt \
-    pkt/*.texi \
-    sp.plantuml.txt \
-    usecases.ru/*.texi \
-    usecases/*.texi
-. ../config
-${MAKEINFO:-makeinfo} \
-    -D "VERSION `cat ../VERSION`" \
-    $MAKEINFO_OPTS \
-    --set-customization-variable SECTION_NAME_IN_TITLE=1 \
-    --set-customization-variable TREE_TRANSFORMATIONS=complete_tree_nodes_menus \
-    --set-customization-variable CLOSE_QUOTE_SYMBOL=\" \
-    --set-customization-variable OPEN_QUOTE_SYMBOL=\" \
-    --output $3 index.texi
+redo-ifchange makeinfo.rc
+MAKEINFO_OPTS="--no-split" . makeinfo.rc
index 8169e99d151747a20094cbacd45700de0775f46d..3944a752f630d5c53d1dd6bc9ef08f22e1994fb2 100644 (file)
@@ -182,7 +182,7 @@ payloads, then send all of remaining in the transport stage.
     Ignore it if it is too nice.
     @item If already downloaded file exists, then queue @emph{DONE}
     sending.
-    @item If @file{.seen} exists, then queue @emph{DONE} sending.
+    @item If @file{seen/XXX} exists, then queue @emph{DONE} sending.
     @item If @file{.part} exists, then queue @emph{FREQ} sending with
     corresponding offset.
     @end itemize
index 14f8ee2b6c4663a363ce20f23de1f0e024847414..61ec8c3513928fbb13eb33941382e5edab309c19 100644 (file)
@@ -50,9 +50,9 @@ non-checksummed (NoCK) @strong{fully} received file. Its checksum is
 verified against its filename either by @ref{nncp-check}, or by working
 online daemons. If it is correct, then its extension is trimmed.
 
-@item LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ.seen
+@item seen/LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ
 @ref{nncp-toss} utility can be invoked with @option{-seen} option,
-leading to creation of @file{.seen} files, telling that the file with
+leading to creation of @file{seen/} files, telling that the file with
 specified hash has already been processed before. It could be useful
 when there are use-cases where multiple ways of packets transfer
 available and there is possibility of duplicates reception. You have to
@@ -60,9 +60,9 @@ manually remove them, when you do not need them (probably because they
 are expired).
 
 @anchor{HdrFile}
-@item LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ.hdr
+@item hdr/LYT64MWSNDK34CVYOO7TA6ZCJ3NWI2OUDBBMX2A4QWF34FIRY4DQ
 If no @ref{CfgNoHdr, nohdr} option is enabled in configuration file,
-then @file{.hdr} files are automatically created for every ordinary
+then @file{hdr/} files are automatically created for every ordinary
 (fully received and checksummed) packet. It literally contains just the
 header of the corresponding packet. It will be automatically created
 even during simple @ref{nncp-stat} call. On filesystems with big
@@ -71,9 +71,9 @@ directories, because it prevents unnecessary read-amplification. On
 other filesystems probably it won't help at all, or even harm
 performance.
 
-There is a hack: you can create more dense @file{.hdr} allocation by
-removing all @file{.hdr} files and then running @command{nncp-stat},
-that will recreate them. In many cases many @file{.hdr} files will be
+There is a hack: you can create more dense @file{hdr/} allocation by
+removing all @file{hdr/} files and then running @command{nncp-stat},
+that will recreate them. In many cases many @file{hdr/} files will be
 allocated more or less linearly on the disk, decreasing listing time
 even more.
 
index 9911395b7c60daad5f3a26fc1682f19ebdb4ad27..52ce4f61fe0eb1016d8effa4118d68718033eff4 100644 (file)
@@ -187,7 +187,7 @@ func main() {
                                        if err = os.Remove(job.Path); err != nil {
                                                log.Fatalln("Error during deletion:", err)
                                        } else if ctx.HdrUsage {
-                                               os.Remove(job.Path + nncp.HdrSuffix)
+                                               os.Remove(nncp.JobPath2Hdr(job.Path))
                                        }
                                }
                                ctx.LogI(
@@ -376,7 +376,7 @@ func main() {
                                        if !*dryRun {
                                                os.Remove(dstPath)
                                                if ctx.HdrUsage {
-                                                       os.Remove(dstPath + nncp.HdrSuffix)
+                                                       os.Remove(nncp.JobPath2Hdr(dstPath))
                                                }
                                        }
                                } else {
@@ -416,7 +416,9 @@ func main() {
                                })
                                continue
                        }
-                       if _, err = os.Stat(dstPath + nncp.SeenSuffix); err == nil || !os.IsNotExist(err) {
+                       if _, err = os.Stat(filepath.Join(
+                               dstDirPath, nncp.SeenDir, pktName,
+                       )); err == nil || !os.IsNotExist(err) {
                                ctx.LogD("bundle-rx-seen", les, func(les nncp.LEs) string {
                                        return logMsg(les) + ": packet already seen"
                                })
index 7d12066ae875df191a4ba742095657d3c2becddd..03a5a1a5c171a760be52a34631671bb7c635bd5d 100644 (file)
@@ -62,7 +62,7 @@ func main() {
                maxOnlineTimeSec  = flag.Uint("maxonlinetime", 0, "Override maxonlinetime option")
 
                autoToss       = flag.Bool("autotoss", false, "Toss after call is finished")
-               autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing")
+               autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing")
                autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing")
                autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing")
                autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing")
index 177065fce845a4b10f7615f64e8a6fd6572a7e12..4024b5be4e07afe34c6f31adcf228492ed1b28ba 100644 (file)
@@ -51,7 +51,7 @@ func main() {
                warranty  = flag.Bool("warranty", false, "Print warranty information")
 
                autoToss       = flag.Bool("autotoss", false, "Toss after call is finished")
-               autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing")
+               autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing")
                autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing")
                autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing")
                autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing")
index 30bdaa9077270f686ae0957b4e360424d0808542..56e4e4f40a7a9a886587e37a41476f0a568eb6b7 100644 (file)
@@ -179,7 +179,7 @@ func main() {
   # umask: "022"
   # Omit progress showing by default
   # noprogress: true
-  # Do not use .hdr files
+  # Do not use hdr/ files
   # nohdr: true
 
   # MultiCast Discovery:
index 266fc7a8f699baa98ef623b65994c3b948db670e..4e9231d942671bd15418f28b69207ae91344fe81 100644 (file)
@@ -124,7 +124,7 @@ func main() {
                warranty  = flag.Bool("warranty", false, "Print warranty information")
 
                autoToss       = flag.Bool("autotoss", false, "Toss after call is finished")
-               autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create .seen files during tossing")
+               autoTossDoSeen = flag.Bool("autotoss-seen", false, "Create seen/ files during tossing")
                autoTossNoFile = flag.Bool("autotoss-nofile", false, "Do not process \"file\" packets during tossing")
                autoTossNoFreq = flag.Bool("autotoss-nofreq", false, "Do not process \"freq\" packets during tossing")
                autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing")
index 39cd0c644e5171c5e0b0af37e1721f460ec74088..b9ca28977c7380834393c7709cc585a9e95a1238 100644 (file)
@@ -54,15 +54,15 @@ func main() {
                cfgPath   = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file")
                doAll     = flag.Bool("all", false, "Apply remove rules to all nodes")
                doTmp     = flag.Bool("tmp", false, "Remove all temporary files")
-               doHdr     = flag.Bool("hdr", false, "Remove all .hdr files")
+               doHdr     = flag.Bool("hdr", false, "Remove all hdr/ files")
                doLock    = flag.Bool("lock", false, "Remove all lock files")
                nodeRaw   = flag.String("node", "", "Node to remove files in")
                doRx      = flag.Bool("rx", false, "Process received packets")
                doTx      = flag.Bool("tx", false, "Process transfered packets")
                doPart    = flag.Bool("part", false, "Remove only .part files")
-               doSeen    = flag.Bool("seen", false, "Remove only .seen files")
+               doSeen    = flag.Bool("seen", false, "Remove only seen/ files")
                doNoCK    = flag.Bool("nock", false, "Remove only .nock files")
-               doArea    = flag.Bool("area", false, "Remove only area/*.seen files")
+               doArea    = flag.Bool("area", false, "Remove only area/* seen files")
                older     = flag.String("older", "", "XXX{smhd}: only older than XXX number of time units")
                dryRun    = flag.Bool("dryrun", false, "Do not actually remove files")
                pktRaw    = flag.String("pkt", "", "Packet to remove")
@@ -204,9 +204,7 @@ func main() {
                                                })
                                                return nil
                                        }
-                                       if (*doSeen && strings.HasSuffix(info.Name(), nncp.SeenSuffix)) ||
-                                               (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) ||
-                                               (*doHdr && strings.HasSuffix(info.Name(), nncp.HdrSuffix)) ||
+                                       if (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) ||
                                                (*doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix)) {
                                                ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg)
                                                if *dryRun {
@@ -233,7 +231,7 @@ func main() {
                                        return nil
                                })
                }
-               if *pktRaw != "" || *doRx || *doSeen || *doNoCK || *doHdr || *doPart {
+               if *pktRaw != "" || *doRx || *doNoCK || *doPart {
                        if err = remove(nncp.TRx); err != nil {
                                log.Fatalln("Can not remove:", err)
                        }
@@ -243,6 +241,63 @@ func main() {
                                log.Fatalln("Can not remove:", err)
                        }
                }
+               removeSub := func(p string, everything bool) error {
+                       return filepath.Walk(
+                               p, func(path string, info os.FileInfo, err error) error {
+                                       if err != nil {
+                                               if os.IsNotExist(err) {
+                                                       return nil
+                                               }
+                                               return err
+                                       }
+                                       if info.IsDir() {
+                                               return nil
+                                       }
+                                       logMsg := func(les nncp.LEs) string {
+                                               return fmt.Sprintf("File %s: removed", path)
+                                       }
+                                       if everything {
+                                               ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg)
+                                               if *dryRun {
+                                                       return nil
+                                               }
+                                               return os.Remove(path)
+                                       }
+                                       if now.Sub(info.ModTime()) < oldBoundary {
+                                               ctx.LogD(
+                                                       "rm-skip", nncp.LEs{{K: "File", V: path}},
+                                                       func(les nncp.LEs) string {
+                                                               return fmt.Sprintf("File %s: too fresh, skipping", path)
+                                                       },
+                                               )
+                                       } else if !*dryRun {
+                                               return os.Remove(path)
+                                       }
+                                       return nil
+                               },
+                       )
+               }
+               if *doRx || *doSeen {
+                       if err = removeSub(filepath.Join(
+                               ctx.Spool, node.Id.String(), string(nncp.TRx), nncp.SeenDir,
+                       ), *doSeen); err != nil {
+                               log.Fatalln("Can not remove:", err)
+                       }
+               }
+               if *doRx || *doHdr {
+                       if err = removeSub(filepath.Join(
+                               ctx.Spool, node.Id.String(), string(nncp.TRx), nncp.HdrDir,
+                       ), *doHdr); err != nil {
+                               log.Fatalln("Can not remove:", err)
+                       }
+               }
+               if *doTx || *doHdr {
+                       if err = removeSub(filepath.Join(
+                               ctx.Spool, node.Id.String(), string(nncp.TTx), nncp.HdrDir,
+                       ), *doHdr); err != nil {
+                               log.Fatalln("Can not remove:", err)
+                       }
+               }
                if *doArea {
                        if err = filepath.Walk(
                                filepath.Join(ctx.Spool, node.Id.String(), nncp.AreaDir),
@@ -262,20 +317,17 @@ func main() {
                                                })
                                                return nil
                                        }
-                                       if strings.HasSuffix(info.Name(), nncp.SeenSuffix) {
-                                               ctx.LogI(
-                                                       "rm",
-                                                       nncp.LEs{{K: "File", V: path}},
-                                                       func(les nncp.LEs) string {
-                                                               return fmt.Sprintf("File %s: removed", path)
-                                                       },
-                                               )
-                                               if *dryRun {
-                                                       return nil
-                                               }
-                                               return os.Remove(path)
+                                       ctx.LogI(
+                                               "rm",
+                                               nncp.LEs{{K: "File", V: path}},
+                                               func(les nncp.LEs) string {
+                                                       return fmt.Sprintf("File %s: removed", path)
+                                               },
+                                       )
+                                       if *dryRun {
+                                               return nil
                                        }
-                                       return nil
+                                       return os.Remove(path)
                                }); err != nil {
                                log.Fatalln("Can not remove:", err)
                        }
index 719b8a4c61cb4022e544a66b9a0a3363c777cdeb..8a3b8c8cbc93d856ab8490d75b5ea0541faf92df 100644 (file)
@@ -42,7 +42,7 @@ func main() {
                nodeRaw   = flag.String("node", "", "Process only that node")
                niceRaw   = flag.String("nice", nncp.NicenessFmt(255), "Minimal required niceness")
                dryRun    = flag.Bool("dryrun", false, "Do not actually write any tossed data")
-               doSeen    = flag.Bool("seen", false, "Create .seen files")
+               doSeen    = flag.Bool("seen", false, "Create seen/ files")
                cycle     = flag.Uint("cycle", 0, "Repeat tossing after N seconds in infinite loop")
                noFile    = flag.Bool("nofile", false, "Do not process \"file\" packets")
                noFreq    = flag.Bool("nofreq", false, "Do not process \"freq\" packets")
index fb7a0e0807e1aa8edb45af08acce79ac65907527..e565ece351e5eb72a525f2a682e5680104828629 100644 (file)
@@ -423,7 +423,9 @@ Tx:
                                })
                                continue
                        }
-                       if _, err = os.Stat(filepath.Join(dstPath, pktName+nncp.SeenSuffix)); err == nil || !os.IsNotExist(err) {
+                       if _, err = os.Stat(filepath.Join(
+                               dstPath, nncp.SeenDir, pktName,
+                       )); err == nil || !os.IsNotExist(err) {
                                ctx.LogD("xfer-tx-seen", les, func(les nncp.LEs) string {
                                        return logMsg(les) + ": already seen"
                                })
@@ -518,7 +520,7 @@ Tx:
                                        })
                                        isBad = true
                                } else if ctx.HdrUsage {
-                                       os.Remove(job.Path + nncp.HdrSuffix)
+                                       os.Remove(nncp.JobPath2Hdr(job.Path))
                                }
                        }
                }
index b1b590b351fa9169805222cdabda34a2276b3a2d..cfbe1b57d3b3e9dbfabcfa8a277d7f198cc9118e 100644 (file)
@@ -69,7 +69,8 @@ func (ctx *Ctx) FindNode(id string) (*Node, error) {
        return node, nil
 }
 
-func ensureDir(p string) error {
+func ensureDir(dirs ...string) error {
+       p := filepath.Join(dirs...)
        fi, err := os.Stat(p)
        if err == nil {
                if fi.IsDir() {
index 6f18f51452d1294839c784d6b1418c8d29756685..7e920fd5c60182609008e3b2457ab32324bba17c 100644 (file)
@@ -35,7 +35,7 @@ const (
        TRx TRxTx = "rx"
        TTx TRxTx = "tx"
 
-       HdrSuffix = ".hdr"
+       HdrDir = "hdr"
 )
 
 type Job struct {
@@ -45,6 +45,10 @@ type Job struct {
        HshValue *[MTHSize]byte
 }
 
+func JobPath2Hdr(jobPath string) string {
+       return filepath.Join(filepath.Dir(jobPath), HdrDir, filepath.Base(jobPath))
+}
+
 func (ctx *Ctx) HdrRead(r io.Reader) (*PktEnc, []byte, error) {
        var pktEnc PktEnc
        _, err := xdr.Unmarshal(r, &pktEnc)
@@ -80,7 +84,13 @@ func (ctx *Ctx) HdrWrite(pktEncRaw []byte, tgt string) error {
                os.Remove(tmpHdr.Name())
                return err
        }
-       if err = os.Rename(tmpHdr.Name(), tgt+HdrSuffix); err != nil {
+       if err = ensureDir(filepath.Dir(tgt), HdrDir); err != nil {
+               ctx.LogE("hdr-write-ensure-mkdir", nil, err, func(les LEs) string {
+                       return "Header writing: ensuring directory"
+               })
+               return err
+       }
+       if err = os.Rename(tmpHdr.Name(), JobPath2Hdr(tgt)); err != nil {
                ctx.LogE("hdr-write-rename", nil, err, func(les LEs) string {
                        return "Header writing: renaming"
                })
@@ -137,7 +147,7 @@ func (ctx *Ctx) jobsFind(nodeId *NodeId, xx TRxTx, nock, part bool) chan Job {
                        if nock || part {
                                fd, err = os.Open(pth)
                        } else {
-                               fd, err = os.Open(pth + HdrSuffix)
+                               fd, err = os.Open(JobPath2Hdr(pth))
                                if err != nil && os.IsNotExist(err) {
                                        hdrExists = false
                                        fd, err = os.Open(pth)
index 1f2e91504dfd7d71c15cb86787fc68afeafb52fb..8c212b3ee20c19c05f5cae0b45af376d261e6fda 100644 (file)
--- a/src/sp.go
+++ b/src/sp.go
@@ -1263,7 +1263,10 @@ func (state *SPState) ProcessSP(payload []byte) ([][]byte, error) {
                                }
                                continue
                        }
-                       if _, err = os.Stat(pktPath + SeenSuffix); err == nil {
+                       if _, err = os.Stat(filepath.Join(
+                               state.Ctx.Spool, state.Node.Id.String(), string(TRx),
+                               SeenDir, Base32Codec.EncodeToString(info.Hash[:]),
+                       )); err == nil {
                                state.Ctx.LogI("sp-info-seen", lesp, func(les LEs) string {
                                        return logMsg(les) + ": already seen"
                                })
@@ -1584,7 +1587,7 @@ func (state *SPState) ProcessSP(payload []byte) ([][]byte, error) {
                                        return fmt.Sprintf("Packet %s is sent", pktName)
                                })
                                if state.Ctx.HdrUsage {
-                                       os.Remove(pth + HdrSuffix)
+                                       os.Remove(JobPath2Hdr(pth))
                                }
                        } else {
                                state.Ctx.LogE("sp-done", lesp, err, logMsg)
index 9b8a6d18b087415eddf61a3dc889b6e42412d1f9..4037831b59618319fdee6083127d91b83af21622 100644 (file)
@@ -43,9 +43,13 @@ import (
 )
 
 const (
-       SeenSuffix = ".seen"
+       SeenDir = "seen"
 )
 
+func jobPath2Seen(jobPath string) string {
+       return filepath.Join(filepath.Dir(jobPath), SeenDir, filepath.Base(jobPath))
+}
+
 func newNotification(fromTo *FromToJSON, subject string, body []byte) io.Reader {
        lines := []string{
                "From: " + fromTo.From,
@@ -198,7 +202,10 @@ func jobProcess(
                })
                if !dryRun && jobPath != "" {
                        if doSeen {
-                               if fd, err := os.Create(jobPath + SeenSuffix); err == nil {
+                               if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil {
+                                       return err
+                               }
+                               if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil {
                                        fd.Close()
                                        if err = DirSync(filepath.Dir(jobPath)); err != nil {
                                                ctx.LogE("rx-dirsync", les, err, func(les LEs) string {
@@ -223,7 +230,7 @@ func jobProcess(
                                })
                                return err
                        } else if ctx.HdrUsage {
-                               os.Remove(jobPath + HdrSuffix)
+                               os.Remove(JobPath2Hdr(jobPath))
                        }
                }
 
@@ -391,7 +398,10 @@ func jobProcess(
                if !dryRun {
                        if jobPath != "" {
                                if doSeen {
-                                       if fd, err := os.Create(jobPath + SeenSuffix); err == nil {
+                                       if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil {
+                                               return err
+                                       }
+                                       if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil {
                                                fd.Close()
                                                if err = DirSync(filepath.Dir(jobPath)); err != nil {
                                                        ctx.LogE("rx-dirsync", les, err, func(les LEs) string {
@@ -416,7 +426,7 @@ func jobProcess(
                                        })
                                        return err
                                } else if ctx.HdrUsage {
-                                       os.Remove(jobPath + HdrSuffix)
+                                       os.Remove(JobPath2Hdr(jobPath))
                                }
                        }
                        if len(sendmail) > 0 && ctx.NotifyFile != nil {
@@ -516,7 +526,10 @@ func jobProcess(
                if !dryRun {
                        if jobPath != "" {
                                if doSeen {
-                                       if fd, err := os.Create(jobPath + SeenSuffix); err == nil {
+                                       if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil {
+                                               return err
+                                       }
+                                       if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil {
                                                fd.Close()
                                                if err = DirSync(filepath.Dir(jobPath)); err != nil {
                                                        ctx.LogE("rx-dirsync", les, err, func(les LEs) string {
@@ -541,7 +554,7 @@ func jobProcess(
                                        })
                                        return err
                                } else if ctx.HdrUsage {
-                                       os.Remove(jobPath + HdrSuffix)
+                                       os.Remove(JobPath2Hdr(jobPath))
                                }
                        }
                        if len(sendmail) > 0 && ctx.NotifyFreq != nil {
@@ -629,7 +642,10 @@ func jobProcess(
                })
                if !dryRun && jobPath != "" {
                        if doSeen {
-                               if fd, err := os.Create(jobPath + SeenSuffix); err == nil {
+                               if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil {
+                                       return err
+                               }
+                               if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil {
                                        fd.Close()
                                        if err = DirSync(filepath.Dir(jobPath)); err != nil {
                                                ctx.LogE("rx-dirsync", les, err, func(les LEs) string {
@@ -655,7 +671,7 @@ func jobProcess(
                                })
                                return err
                        } else if ctx.HdrUsage {
-                               os.Remove(jobPath + HdrSuffix)
+                               os.Remove(JobPath2Hdr(jobPath))
                        }
                }
 
@@ -698,7 +714,7 @@ func jobProcess(
                                seenDir := filepath.Join(
                                        ctx.Spool, nodeId.String(), AreaDir, area.Id.String(),
                                )
-                               seenPath := filepath.Join(seenDir, msgHash+SeenSuffix)
+                               seenPath := filepath.Join(seenDir, msgHash)
                                logMsgNode := func(les LEs) string {
                                        return fmt.Sprintf(
                                                "%s: echoing to: %s", logMsg(les), node.Name,
@@ -719,7 +735,7 @@ func jobProcess(
                                seenDir := filepath.Join(
                                        ctx.Spool, nodeId.String(), AreaDir, area.Id.String(),
                                )
-                               seenPath := filepath.Join(seenDir, msgHash+SeenSuffix)
+                               seenPath := filepath.Join(seenDir, msgHash)
                                logMsgNode := func(les LEs) string {
                                        return fmt.Sprintf("%s: echo to: %s", logMsg(les), node.Name)
                                }
@@ -759,7 +775,7 @@ func jobProcess(
                seenDir := filepath.Join(
                        ctx.Spool, ctx.SelfId.String(), AreaDir, area.Id.String(),
                )
-               seenPath := filepath.Join(seenDir, msgHash+SeenSuffix)
+               seenPath := filepath.Join(seenDir, msgHash)
                if _, err := os.Stat(seenPath); err == nil {
                        ctx.LogD("rx-area-seen", les, func(les LEs) string {
                                return logMsg(les) + ": already seen"
@@ -776,7 +792,7 @@ func jobProcess(
                                        })
                                        return err
                                } else if ctx.HdrUsage {
-                                       os.Remove(jobPath + HdrSuffix)
+                                       os.Remove(JobPath2Hdr(jobPath))
                                }
                        }
                        return nil
@@ -876,7 +892,7 @@ func jobProcess(
                                })
                                return err
                        } else if ctx.HdrUsage {
-                               os.Remove(jobPath + HdrSuffix)
+                               os.Remove(JobPath2Hdr(jobPath))
                        }
                }
 
index 2b166d1d49b546999acf8085de99cf28fdf3f3ce..e096ed65f873ddeb48f9ac635a01bbeea8849ba2 100644 (file)
--- a/src/tx.go
+++ b/src/tx.go
@@ -239,7 +239,7 @@ func (ctx *Ctx) Tx(
                seenDir := filepath.Join(
                        ctx.Spool, ctx.SelfId.String(), AreaDir, areaId.String(),
                )
-               seenPath := filepath.Join(seenDir, msgHash+SeenSuffix)
+               seenPath := filepath.Join(seenDir, msgHash)
                les := LEs{
                        {"Node", node.Id},
                        {"Nice", int(nice)},