Pyshop migration script

diff --git a/gocheese.texi b/gocheese.texi
index fac8df0e4c564c3652a1e4043224be30c5dc4f38..a1884397257cab203d2361e59de005084e41a6ac 100644 (file)
--- a/gocheese.texi
+++ b/gocheese.texi
@@ -10,9 +10,9 @@ GoCheese is Python private package repository and caching proxy.
 It serves two purposes:
 
 @itemize
-@item hosting of private locally uploaded packages (conforming to
-    @url{https://www.python.org/dev/peps/pep-0503/, PEP-0503} (Simple
-    Repository API))
+@item hosting of private locally uploaded packages
+    (conforming to @url{https://www.python.org/dev/peps/pep-0503/, PEP-0503}
+    (Simple Repository API))
 @item proxying and caching of missing packages from upstream
     @url{https://pypi.org/, PyPI}
 @end itemize
@@ -30,6 +30,10 @@ but nearly all the code was rewritten. It has huge differences:
 @item no package overwriting ability (as PyPI does)
 @end itemize
 
+Also it contains @file{pyshop2packages.sh} migration script for
+converting @url{https://pypi.org/project/pyshop/, Pyshop} database into
+GoCheese one, including private packages.
+
 GoCheese is free software, licenced under
 @url{https://www.gnu.org/licenses/gpl-3.0.html, GNU GPLv3}:
 see the file COPYING for copying conditions.

diff --git a/pyshop2packages.sh b/pyshop2packages.sh
new file mode 100755 (executable)
index 0000000..c0ae7a5
--- /dev/null
+++ b/pyshop2packages.sh
@@ -0,0 +1,57 @@
+#!/bin/sh -e
+# Convert Pyshop working directory to GoCheese friendly one.
+# Script expects to be run in packages repository (a/, b/, ...)
+# with pyshop.db database file. It will create packages/ directory
+# with all Pyshop-downloaded/uploaded files, SHA256 checksums,
+# private package marks. Also it will call GoCheese's /simple/ API
+# for forcing metainformation update (necessary for storing SHA256
+# checksums missing in Pyshop).
+
+pkgname() {
+    perl -ne "s/[-_.]+/-/g ; print lc"
+}
+
+########################################################################
+# Copy all already existing files
+########################################################################
+ctr=0
+echo "
+SELECT package.name, release_file.filename
+FROM release_file
+JOIN release ON release.id = release_file.release_id
+JOIN package ON release.package_id = package.id
+ORDER BY package.name
+" | sqlite3 --separator "      " pyshop.db | while read pkginfo ; do
+    ctr=$(( $ctr + 1 ))
+    [ $(( $ctr % 100 )) -ne 0 ] || echo $ctr $pkginfo
+    pkg=$(echo "$pkginfo" | cut -f1 | pkgname)
+    filename=$(echo "$pkginfo" | cut -f2)
+    [ -n "$pkg" ]
+    [ -n "$filename" ]
+    src=$(echo $pkg | cut -c1)/$filename
+    dst=packages/$pkg/$filename 
+    [ -r $src ] || continue
+    [ -r $dst ] && continue || :
+    mkdir -p packages/$pkg
+    tee $dst < $src | sha256 | xxd -r -p > $dst.sha256
+done
+
+########################################################################
+# Mark all private packages
+########################################################################
+for pkg in $(echo "SELECT name FROM package WHERE local = true" | sqlite3 pyshop.db); do
+    touch packages/$(echo $pkg | pkgname)/.private
+done
+
+########################################################################
+# Force all metainformation update from upstream
+########################################################################
+cd packages
+for pkg in * ; do
+    curl http://localhost:8080/simple/$pkg/ > /dev/null
+done
+
+########################################################################
+# Assure ckecksums are good
+########################################################################
+gocheese -fsck