From: Sergey Matveev Date: Fri, 6 Dec 2019 21:35:48 +0000 (+0300) Subject: Multiple upstream digest algorithms support X-Git-Tag: v2.1.0~4 X-Git-Url: http://www.git.cypherpunks.ru/?p=gocheese.git;a=commitdiff_plain;h=e396b31d6baa52055648be298dbb33ca91bdf33d Multiple upstream digest algorithms support --- diff --git a/gocheese.go b/gocheese.go index c2a4d0b..ff76c87 100644 --- a/gocheese.go +++ b/gocheese.go @@ -20,12 +20,16 @@ along with this program. If not, see . package main import ( + "bufio" "bytes" "context" + "crypto/md5" "crypto/sha256" + "crypto/sha512" "encoding/hex" "flag" "fmt" + "hash" "io" "io/ioutil" "log" @@ -54,8 +58,6 @@ const ( ` HTMLEnd = " \n\n" HTMLElement = " %s
\n" - SHA256Prefix = "sha256=" - SHA256Ext = ".sha256" InternalFlag = ".internal" GPGSigExt = ".asc" GPGSigAttr = " data-gpg-sig=true" @@ -94,6 +96,7 @@ var ( killed bool + pypiURLParsed *url.URL normalizationRe *regexp.Regexp = regexp.MustCompilePOSIX("[-_.]+") ) @@ -148,23 +151,64 @@ func refreshDir( uri = submatches[1] filename = submatches[2] if pkgURL, err = url.Parse(uri); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusBadGateway) + return false + } + + if pkgURL.Fragment == "" { + log.Println(r.RemoteAddr, "pypi", filename, "no digest provided") + http.Error(w, "no digest provided", http.StatusBadGateway) return false } - if !strings.HasPrefix(pkgURL.Fragment, SHA256Prefix) { - log.Println(r.RemoteAddr, "pypi", filename, "no SHA256 digest provided") - http.Error(w, "no SHA256 digest provided", http.StatusBadGateway) + digestInfo := strings.Split(pkgURL.Fragment, "=") + if len(digestInfo) == 1 { + // Ancient non PEP-0503 PyPIs, assume MD5 + digestInfo = []string{"md5", digestInfo[0]} + } else if len(digestInfo) != 2 { + log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided") + http.Error(w, "invalid digest provided", http.StatusBadGateway) return false } - digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix)) + digest, err = hex.DecodeString(digestInfo[1]) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false } + var hasherNew func() hash.Hash + var hashExt string + var hashSize int + switch digestInfo[0] { + case "md5": + hashExt = ".md5" + hasherNew = md5.New + hashSize = md5.Size + case "sha256": + hashExt = ".sha256" + hasherNew = sha256.New + hashSize = sha256.Size + case "sha512": + hashExt = ".sha512" + hasherNew = sha512.New + hashSize = sha512.Size + default: + log.Println( + r.RemoteAddr, "pypi", filename, + "unknown digest algorithm", digestInfo[0], + ) + http.Error(w, "unknown digest algorithm", http.StatusBadGateway) + return false + } + if len(digest) != hashSize { + log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length") + http.Error(w, "invalid digest length", http.StatusBadGateway) + return false + } + pkgURL.Fragment = "" - uri = pkgURL.String() if pkgURL.Host == "" { - uri = *pypiURL + strings.TrimPrefix(uri, "/") + uri = pypiURLParsed.ResolveReference(pkgURL).String() + } else { + uri = pkgURL.String() } path = filepath.Join(dirPath, filename) if filename == filenameGet { @@ -181,19 +225,31 @@ func refreshDir( return false } defer resp.Body.Close() - hasher := sha256.New() + hasher := hasherNew() + hasherOur := sha256.New() dst, err := TempFile(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } - wr := io.MultiWriter(hasher, dst) + dstBuf := bufio.NewWriter(dst) + wrs := []io.Writer{hasher, dstBuf} + if hashExt != ".sha256" { + wrs = append(wrs, hasherOur) + } + wr := io.MultiWriter(wrs...) if _, err = io.Copy(wr, resp.Body); err != nil { os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return false } + if err = dstBuf.Flush(); err != nil { + os.Remove(dst.Name()) + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } if bytes.Compare(hasher.Sum(nil), digest) != 0 { log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch") os.Remove(dst.Name()) @@ -207,7 +263,10 @@ func refreshDir( http.Error(w, err.Error(), http.StatusInternalServerError) return false } - dst.Close() + if err = dst.Close(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } if err = os.Rename(dst.Name(), path); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false @@ -216,6 +275,10 @@ func refreshDir( http.Error(w, err.Error(), http.StatusInternalServerError) return false } + if hashExt != ".sha256" { + hashExt = ".sha256" + digest = hasherOur.Sum(nil) + } } if filename == filenameGet || gpgUpdate { if _, err = os.Stat(path); err != nil { @@ -241,7 +304,7 @@ func refreshDir( log.Println(r.RemoteAddr, "pypi downloaded signature", filename) } GPGSigSkip: - path = path + SHA256Ext + path = path + hashExt _, err = os.Stat(path) if err == nil { continue @@ -295,45 +358,60 @@ func listDir( } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) { return } - files, err := ioutil.ReadDir(dirPath) + fis, err := ioutil.ReadDir(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } var result bytes.Buffer result.WriteString(fmt.Sprintf(HTMLBegin, dir)) - var data []byte + var digest []byte var gpgSigAttr string - var filenameClean string - for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { - continue - } - if killed { - // Skip expensive I/O when shutting down - http.Error(w, "shutting down", http.StatusInternalServerError) - return - } - data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name())) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + var fnClean string + files := make(map[string]struct{}, len(fis)/2) + for _, fi := range fis { + files[fi.Name()] = struct{}{} + } + for _, algoExt := range []string{".sha256", ".sha512", ".md5"} { + for fn, _ := range files { + if killed { + // Skip expensive I/O when shutting down + http.Error(w, "shutting down", http.StatusInternalServerError) + return + } + if !strings.HasSuffix(fn, algoExt) { + continue + } + digest, err = ioutil.ReadFile(filepath.Join(dirPath, fn)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + fnClean = strings.TrimSuffix(fn, algoExt) + if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); os.IsNotExist(err) { + gpgSigAttr = "" + } else { + gpgSigAttr = GPGSigAttr + } + result.WriteString(fmt.Sprintf( + HTMLElement, + strings.Join([]string{ + *refreshURLPath, dir, "/", fnClean, + "#", algoExt[1:], "=", hex.EncodeToString(digest), + }, ""), + gpgSigAttr, + fnClean, + )) + for _, n := range []string{ + fnClean, + fnClean + GPGSigExt, + fnClean + ".sha256", + fnClean + ".sha512", + fnClean + ".md5", + } { + delete(files, n) + } } - filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext) - if _, err = os.Stat(filepath.Join(dirPath, filenameClean+GPGSigExt)); os.IsNotExist(err) { - gpgSigAttr = "" - } else { - gpgSigAttr = GPGSigAttr - } - result.WriteString(fmt.Sprintf( - HTMLElement, - strings.Join([]string{ - *refreshURLPath, dir, "/", - filenameClean, "#", SHA256Prefix, hex.EncodeToString(data), - }, ""), - gpgSigAttr, - filenameClean, - )) } result.WriteString(HTMLEnd) w.Write(result.Bytes()) @@ -419,6 +497,7 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + dstBuf := bufio.NewWriter(dst) hasher := sha256.New() wr := io.MultiWriter(hasher, dst) if _, err = io.Copy(wr, src); err != nil { @@ -427,6 +506,12 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + if err = dstBuf.Flush(); err != nil { + os.Remove(dst.Name()) + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } if err = dst.Sync(); err != nil { os.Remove(dst.Name()) dst.Close() @@ -453,7 +538,7 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if err = WriteFileSync(dirPath, path+SHA256Ext, digest); err != nil { + if err = WriteFileSync(dirPath, path+".sha256", digest); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -546,10 +631,10 @@ func goodIntegrity() bool { log.Fatal(err) } for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { + if !strings.HasSuffix(file.Name(), ".sha256") { continue } - pkgName = strings.TrimSuffix(file.Name(), SHA256Ext) + pkgName = strings.TrimSuffix(file.Name(), ".sha256") data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName)) if err != nil { if os.IsNotExist(err) { @@ -597,6 +682,11 @@ func main() { if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") { log.Fatalln("Both -tls-cert and -tls-key are required") } + var err error + pypiURLParsed, err = url.Parse(*pypiURL) + if err != nil { + log.Fatalln(err) + } refreshPasswd() log.Println("root:", *root, "bind:", *bind) diff --git a/gocheese.texi b/gocheese.texi index 26ab178..6cff768 100644 --- a/gocheese.texi +++ b/gocheese.texi @@ -183,7 +183,8 @@ Root directory has the following hierarchy: @verbatim root +-- public-package - | +- public-package-0.1.tar.gz.sha256 + | +- public-package-0.1.tar.gz.md5 + | +- public-package-0.1.1.tar.gz.sha256 | +- public-package-0.2.tar.gz | +- public-package-0.2.tar.gz.asc | +- public-package-0.2.tar.gz.sha256