Store blake2b_256 during refresh

Use its value from JSON metadata if available.
BLAKE2b is much faster on 64-bit architectures,
so it is much more preferable in practice.

diff --git a/refresh.go b/refresh.go
index e60f7d9a864eb9284bd012987289db1f4856d276..9aa4849e2e06a88f401bb1b8b0586db992c23529 100644 (file)
--- a/refresh.go
+++ b/refresh.go
@@ -42,8 +42,8 @@ import (
 )
 
 const (
-       HashAlgoSHA256     = "sha256"
        HashAlgoBLAKE2b256 = "blake2b_256"
+       HashAlgoSHA256     = "sha256"
        HashAlgoSHA512     = "sha512"
        HashAlgoMD5        = "md5"
        InternalFlag       = ".internal"
@@ -54,8 +54,8 @@ var (
        PyPIURLParsed     *url.URL
        PyPIHTTPTransport http.Transport
        KnownHashAlgos    []string = []string{
-               HashAlgoSHA256,
                HashAlgoBLAKE2b256,
+               HashAlgoSHA256,
                HashAlgoSHA512,
                HashAlgoMD5,
        }
@@ -248,6 +248,9 @@ func refreshDir(
                }
        }
        mtimes := make(map[string]time.Time)
+       digestsBLAKE2b256 := make(map[string][]byte)
+       digestsSHA256 := make(map[string][]byte)
+       digestsSHA512 := make(map[string][]byte)
        for _, releases := range allReleases {
                for _, rel := range releases {
                        if rel.Filename == "" || rel.UploadTimeISO8601 == "" {
@@ -263,6 +266,39 @@ func refreshDir(
                                return false
                        }
                        mtimes[rel.Filename] = t.Truncate(time.Second)
+                       if d := rel.Digests[HashAlgoBLAKE2b256]; d != "" {
+                               digestsBLAKE2b256[rel.Filename], err = hex.DecodeString(d)
+                               if err != nil {
+                                       log.Println(
+                                               "error", r.RemoteAddr, "refresh-json", pkgName,
+                                               "can not decode blake2b_256 digest:", err,
+                                       )
+                                       http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+                                       return false
+                               }
+                       }
+                       if d := rel.Digests[HashAlgoSHA256]; d != "" {
+                               digestsSHA256[rel.Filename], err = hex.DecodeString(d)
+                               if err != nil {
+                                       log.Println(
+                                               "error", r.RemoteAddr, "refresh-json", pkgName,
+                                               "can not decode sha256 digest:", err,
+                                       )
+                                       http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+                                       return false
+                               }
+                       }
+                       if d := rel.Digests[HashAlgoSHA512]; d != "" {
+                               digestsSHA512[rel.Filename], err = hex.DecodeString(d)
+                               if err != nil {
+                                       log.Println(
+                                               "error", r.RemoteAddr, "refresh-json", pkgName,
+                                               "can not decode sha512 digest:", err,
+                                       )
+                                       http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+                                       return false
+                               }
+                       }
                }
        }
 
@@ -305,56 +341,73 @@ func refreshDir(
                        return false
                }
 
-               if pkgURL.Fragment == "" {
-                       log.Println(r.RemoteAddr, "pypi", filename, "no digest")
-                       http.Error(w, "no digest provided", http.StatusBadGateway)
-                       return false
-               }
-               digestInfo := strings.Split(pkgURL.Fragment, "=")
-               if len(digestInfo) == 1 {
-                       // Ancient non PEP-0503 PyPIs, assume MD5
-                       digestInfo = []string{"md5", digestInfo[0]}
-               } else if len(digestInfo) != 2 {
-                       log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
-                       http.Error(w, "invalid digest provided", http.StatusBadGateway)
-                       return false
-               }
-               digest, err := hex.DecodeString(digestInfo[1])
-               if err != nil {
-                       log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
-                       http.Error(w, err.Error(), http.StatusBadGateway)
-                       return false
-               }
-               hashAlgo := digestInfo[0]
+               var hashAlgo string
                var hasherNew func() hash.Hash
-               var hashSize int
-               switch hashAlgo {
-               case HashAlgoMD5:
-                       hasherNew = md5.New
-                       hashSize = md5.Size
-               case HashAlgoSHA256:
+               var digest []byte
+               if d := digestsBLAKE2b256[filename]; d != nil {
+                       hasherNew = blake2b256New
+                       hashAlgo = HashAlgoBLAKE2b256
+                       digest = d
+               } else if d := digestsSHA256[filename]; d != nil {
                        hasherNew = sha256.New
-                       hashSize = sha256.Size
-               case HashAlgoSHA512:
+                       hashAlgo = HashAlgoSHA256
+                       digest = d
+               } else if d := digestsSHA512[filename]; d != nil {
                        hasherNew = sha512.New
-                       hashSize = sha512.Size
-               case HashAlgoBLAKE2b256:
-                       hasherNew = blake2b256New
-                       hashSize = blake2b.Size256
-               default:
-                       log.Println(
-                               "error", r.RemoteAddr, "pypi",
-                               filename, "unknown digest", hashAlgo,
-                       )
-                       http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
-                       return false
-               }
-               if len(digest) != hashSize {
-                       log.Println(
-                               "error", r.RemoteAddr, "pypi",
-                               filename, "invalid digest length")
-                       http.Error(w, "invalid digest length", http.StatusBadGateway)
-                       return false
+                       hashAlgo = HashAlgoSHA512
+                       digest = d
+               } else {
+                       if pkgURL.Fragment == "" {
+                               log.Println(r.RemoteAddr, "pypi", filename, "no digest")
+                               http.Error(w, "no digest provided", http.StatusBadGateway)
+                               return false
+                       }
+                       digestInfo := strings.Split(pkgURL.Fragment, "=")
+                       if len(digestInfo) == 1 {
+                               // Ancient non PEP-0503 PyPIs, assume MD5
+                               digestInfo = []string{"md5", digestInfo[0]}
+                       } else if len(digestInfo) != 2 {
+                               log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
+                               http.Error(w, "invalid digest provided", http.StatusBadGateway)
+                               return false
+                       }
+                       var err error
+                       digest, err = hex.DecodeString(digestInfo[1])
+                       if err != nil {
+                               log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
+                               http.Error(w, err.Error(), http.StatusBadGateway)
+                               return false
+                       }
+                       hashAlgo = digestInfo[0]
+                       var hashSize int
+                       switch hashAlgo {
+                       case HashAlgoBLAKE2b256:
+                               hasherNew = blake2b256New
+                               hashSize = blake2b.Size256
+                       case HashAlgoSHA256:
+                               hasherNew = sha256.New
+                               hashSize = sha256.Size
+                       case HashAlgoSHA512:
+                               hasherNew = sha512.New
+                               hashSize = sha512.Size
+                       case HashAlgoMD5:
+                               hasherNew = md5.New
+                               hashSize = md5.Size
+                       default:
+                               log.Println(
+                                       "error", r.RemoteAddr, "pypi",
+                                       filename, "unknown digest", hashAlgo,
+                               )
+                               http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
+                               return false
+                       }
+                       if len(digest) != hashSize {
+                               log.Println(
+                                       "error", r.RemoteAddr, "pypi",
+                                       filename, "invalid digest length")
+                               http.Error(w, "invalid digest length", http.StatusBadGateway)
+                               return false
+                       }
                }
 
                pkgURL.Fragment = ""
@@ -393,8 +446,8 @@ func refreshDir(
                                return false
                        }
                        hasher := hasherNew()
-                       hasherSHA256 := sha256.New()
                        hasherBLAKE2b256 := blake2b256New()
+                       hasherSHA256 := sha256.New()
                        dst, err := TempFile(dirPath)
                        if err != nil {
                                log.Println("error", r.RemoteAddr, "pypi", filename, err)
@@ -403,12 +456,12 @@ func refreshDir(
                        }
                        dstBuf := bufio.NewWriter(dst)
                        wrs := []io.Writer{hasher, dstBuf}
-                       if hashAlgo != HashAlgoSHA256 {
-                               wrs = append(wrs, hasherSHA256)
-                       }
                        if hashAlgo != HashAlgoBLAKE2b256 {
                                wrs = append(wrs, hasherBLAKE2b256)
                        }
+                       if hashAlgo != HashAlgoSHA256 {
+                               wrs = append(wrs, hasherSHA256)
+                       }
                        wr := io.MultiWriter(wrs...)
                        if _, err = io.Copy(wr, resp.Body); err != nil {
                                os.Remove(dst.Name())
@@ -469,36 +522,36 @@ func refreshDir(
                                return false
                        }
 
-                       var digestSHA256 []byte
                        var digestBLAKE2b256 []byte
-                       if hashAlgo == HashAlgoSHA256 {
-                               digestSHA256 = hasher.Sum(nil)
-                       } else {
-                               digestSHA256 = hasherSHA256.Sum(nil)
-                       }
+                       var digestSHA256 []byte
                        if hashAlgo == HashAlgoBLAKE2b256 {
                                digestBLAKE2b256 = hasher.Sum(nil)
                        } else {
                                digestBLAKE2b256 = hasherBLAKE2b256.Sum(nil)
                        }
+                       if hashAlgo == HashAlgoSHA256 {
+                               digestSHA256 = hasher.Sum(nil)
+                       } else {
+                               digestSHA256 = hasherSHA256.Sum(nil)
+                       }
                        if err = WriteFileSync(
-                               dirPath, path+"."+HashAlgoSHA256,
-                               digestSHA256, mtime,
+                               dirPath, path+"."+HashAlgoBLAKE2b256,
+                               digestBLAKE2b256, mtime,
                        ); err != nil {
                                log.Println(
                                        "error", r.RemoteAddr, "pypi",
-                                       path+"."+HashAlgoSHA256, err,
+                                       path+"."+HashAlgoBLAKE2b256, err,
                                )
                                http.Error(w, err.Error(), http.StatusInternalServerError)
                                return false
                        }
                        if err = WriteFileSync(
-                               dirPath, path+"."+HashAlgoBLAKE2b256,
-                               digestBLAKE2b256, mtime,
+                               dirPath, path+"."+HashAlgoSHA256,
+                               digestSHA256, mtime,
                        ); err != nil {
                                log.Println(
                                        "error", r.RemoteAddr, "pypi",
-                                       path+"."+HashAlgoBLAKE2b256, err,
+                                       path+"."+HashAlgoSHA256, err,
                                )
                                http.Error(w, err.Error(), http.StatusInternalServerError)
                                return false