Example downloading scripts

[nncp.git] / doc / warcer.sh

diff --git a/doc/warcer.sh b/doc/warcer.sh
new file mode 100755 (executable)
index 0000000..9448592
--- /dev/null
+++ b/doc/warcer.sh
@@ -0,0 +1,24 @@
+#!/bin/sh -ex
+
+name="$1"-$(date '+%Y%M%d%H%m%S')
+read cmdline
+
+tmp=$(mktemp -d)
+cd $tmp
+wget \
+    --page-requisites \
+    --convert-links \
+    --adjust-extension \
+    --restrict-file-names=ascii \
+    --span-hosts \
+    --random-wait \
+    --execute robots=off \
+    --reject '*.woff*,*.ttf,*.eot,*.js' \
+    --tries 10 \
+    --warc-file "$name" \
+    --no-warc-compression \
+    --no-warc-keep-log \
+    $cmdline || :
+xz -9 "$name".warc
+nncp-file -nice $NNCP_NICE "$name".warc.xz $NNCP_SENDER:
+rm -r $tmp