X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=gocheese.go;h=d6a1a70ba708b63911e02b5bfc55f5d79290aa84;hb=11d218004e3a2668985a6d9b2628cb4b3fdc0051;hp=4b47f3c51a72deda900a457e07f9e59166d184a8;hpb=4b49f65744b823da2903583ab348167a1e2756ac;p=gocheese.git diff --git a/gocheese.go b/gocheese.go index 4b47f3c..d6a1a70 100644 --- a/gocheese.go +++ b/gocheese.go @@ -20,12 +20,16 @@ along with this program. If not, see . package main import ( + "bufio" "bytes" "context" + "crypto/md5" "crypto/sha256" + "crypto/sha512" "encoding/hex" "flag" "fmt" + "hash" "io" "io/ioutil" "log" @@ -41,6 +45,7 @@ import ( "syscall" "time" + "golang.org/x/crypto/blake2b" "golang.org/x/net/netutil" ) @@ -54,11 +59,8 @@ const ( ` HTMLEnd = " \n\n" HTMLElement = " %s
\n" - SHA256Prefix = "sha256=" - SHA256Ext = ".sha256" InternalFlag = ".internal" GPGSigExt = ".asc" - GPGSigAttr = " data-gpg-sig=true" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -74,8 +76,19 @@ along with this program. If not, see .` ) var ( - pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) - Version string = "UNKNOWN" + pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) + normalizationRe = regexp.MustCompilePOSIX("[-_.]+") + + HashAlgoSHA256 = "sha256" + HashAlgoBLAKE2b256 = "blake2_256" + HashAlgoSHA512 = "sha512" + HashAlgoMD5 = "md5" + knownHashAlgos []string = []string{ + HashAlgoSHA256, + HashAlgoBLAKE2b256, + HashAlgoSHA512, + HashAlgoMD5, + } root = flag.String("root", "./packages", "Path to packages directory") bind = flag.String("bind", "[::]:8080", "Address to bind to") @@ -92,7 +105,9 @@ var ( version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") - killed bool + Version string = "UNKNOWN" + killed bool + pypiURLParsed *url.URL ) func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { @@ -107,6 +122,14 @@ func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { return true } +func blake2b256New() hash.Hash { + h, err := blake2b.New256(nil) + if err != nil { + panic(err) + } + return h +} + func refreshDir( w http.ResponseWriter, r *http.Request, @@ -132,30 +155,76 @@ func refreshDir( return false } dirPath := filepath.Join(*root, dir) - var submatches []string - var uri string - var filename string - var path string - var pkgURL *url.URL - var digest []byte for _, lineRaw := range bytes.Split(body, []byte("\n")) { - submatches = pkgPyPI.FindStringSubmatch(string(lineRaw)) + submatches := pkgPyPI.FindStringSubmatch(string(lineRaw)) if len(submatches) == 0 { continue } - uri = submatches[1] - filename = submatches[2] - if pkgURL, err = url.Parse(uri); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + uri := submatches[1] + filename := submatches[2] + pkgURL, err := url.Parse(uri) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return false + } + + if pkgURL.Fragment == "" { + log.Println(r.RemoteAddr, "pypi", filename, "no digest provided") + http.Error(w, "no digest provided", http.StatusBadGateway) return false } - digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix)) + digestInfo := strings.Split(pkgURL.Fragment, "=") + if len(digestInfo) == 1 { + // Ancient non PEP-0503 PyPIs, assume MD5 + digestInfo = []string{"md5", digestInfo[0]} + } else if len(digestInfo) != 2 { + log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided") + http.Error(w, "invalid digest provided", http.StatusBadGateway) + return false + } + digest, err := hex.DecodeString(digestInfo[1]) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false } + hashAlgo := digestInfo[0] + var hasherNew func() hash.Hash + var hashSize int + switch hashAlgo { + case HashAlgoMD5: + hasherNew = md5.New + hashSize = md5.Size + case HashAlgoSHA256: + hasherNew = sha256.New + hashSize = sha256.Size + case HashAlgoSHA512: + hasherNew = sha512.New + hashSize = sha512.Size + case HashAlgoBLAKE2b256: + hasherNew = blake2b256New + hashSize = blake2b.Size256 + default: + log.Println( + r.RemoteAddr, "pypi", filename, + "unknown digest algorithm", hashAlgo, + ) + http.Error(w, "unknown digest algorithm", http.StatusBadGateway) + return false + } + if len(digest) != hashSize { + log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length") + http.Error(w, "invalid digest length", http.StatusBadGateway) + return false + } + pkgURL.Fragment = "" - path = filepath.Join(dirPath, filename) + if pkgURL.Host == "" { + uri = pypiURLParsed.ResolveReference(pkgURL).String() + } else { + uri = pkgURL.String() + } + + path := filepath.Join(dirPath, filename) if filename == filenameGet { if killed { // Skip heavy remote call, when shutting down @@ -163,30 +232,43 @@ func refreshDir( return false } log.Println(r.RemoteAddr, "pypi download", filename) - resp, err = http.Get(pkgURL.String()) + resp, err = http.Get(uri) if err != nil { + log.Println(r.RemoteAddr, "pypi download error:", err.Error()) http.Error(w, err.Error(), http.StatusBadGateway) return false } defer resp.Body.Close() - hasher := sha256.New() + hasher := hasherNew() + hasherSHA256 := sha256.New() dst, err := TempFile(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } - wr := io.MultiWriter(hasher, dst) + dstBuf := bufio.NewWriter(dst) + wrs := []io.Writer{hasher, dstBuf} + if hashAlgo != HashAlgoSHA256 { + wrs = append(wrs, hasherSHA256) + } + wr := io.MultiWriter(wrs...) if _, err = io.Copy(wr, resp.Body); err != nil { os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return false } + if err = dstBuf.Flush(); err != nil { + os.Remove(dst.Name()) + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } if bytes.Compare(hasher.Sum(nil), digest) != 0 { log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch") os.Remove(dst.Name()) dst.Close() - http.Error(w, err.Error(), http.StatusBadGateway) + http.Error(w, "digest mismatch", http.StatusBadGateway) return false } if err = dst.Sync(); err != nil { @@ -195,28 +277,55 @@ func refreshDir( http.Error(w, err.Error(), http.StatusInternalServerError) return false } - dst.Close() + if err = dst.Close(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } if err = os.Rename(dst.Name(), path); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } + if err = DirSync(dirPath); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } + if hashAlgo != HashAlgoSHA256 { + hashAlgo = HashAlgoSHA256 + digest = hasherSHA256.Sum(nil) + for _, algo := range knownHashAlgos[1:] { + os.Remove(path + "." + algo) + } + } } if filename == filenameGet || gpgUpdate { - if _, err = os.Stat(path); err == nil { - if resp, err := http.Get(pkgURL.String() + GPGSigExt); err == nil { - sig, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err == nil { - if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi downloaded signature", filename) - } - } + if _, err = os.Stat(path); err != nil { + goto GPGSigSkip } + resp, err := http.Get(uri + GPGSigExt) + if err != nil { + goto GPGSigSkip + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + goto GPGSigSkip + } + sig, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + goto GPGSigSkip + } + if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) { + log.Println(r.RemoteAddr, "pypi non PGP signature", filename) + goto GPGSigSkip + } + if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } + log.Println(r.RemoteAddr, "pypi downloaded signature", filename) } - path = path + SHA256Ext + GPGSigSkip: + path = path + "." + hashAlgo _, err = os.Stat(path) if err == nil { continue @@ -270,45 +379,50 @@ func listDir( } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) { return } - files, err := ioutil.ReadDir(dirPath) + fis, err := ioutil.ReadDir(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } + files := make(map[string]struct{}, len(fis)/2) + for _, fi := range fis { + files[fi.Name()] = struct{}{} + } var result bytes.Buffer result.WriteString(fmt.Sprintf(HTMLBegin, dir)) - var data []byte - var gpgSigAttr string - var filenameClean string - for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { - continue - } - if killed { - // Skip expensive I/O when shutting down - http.Error(w, "shutting down", http.StatusInternalServerError) - return - } - data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name())) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + for _, algo := range knownHashAlgos { + for fn, _ := range files { + if killed { + // Skip expensive I/O when shutting down + http.Error(w, "shutting down", http.StatusInternalServerError) + return + } + if !strings.HasSuffix(fn, "."+algo) { + continue + } + delete(files, fn) + digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + fnClean := strings.TrimSuffix(fn, "."+algo) + delete(files, fnClean) + gpgSigAttr := "" + if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil { + gpgSigAttr = " data-gpg-sig=true" + delete(files, fnClean+GPGSigExt) + } + result.WriteString(fmt.Sprintf( + HTMLElement, + strings.Join([]string{ + *refreshURLPath, dir, "/", fnClean, + "#", algo, "=", hex.EncodeToString(digest), + }, ""), + gpgSigAttr, + fnClean, + )) } - filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext) - if _, err = os.Stat(filepath.Join(dirPath, filenameClean+GPGSigExt)); os.IsNotExist(err) { - gpgSigAttr = "" - } else { - gpgSigAttr = GPGSigAttr - } - result.WriteString(fmt.Sprintf( - HTMLElement, - strings.Join([]string{ - *refreshURLPath, dir, "/", - filenameClean, "#", SHA256Prefix, hex.EncodeToString(data), - }, ""), - gpgSigAttr, - filenameClean, - )) } result.WriteString(HTMLEnd) w.Write(result.Bytes()) @@ -326,6 +440,7 @@ func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { } func serveUpload(w http.ResponseWriter, r *http.Request) { + // Authentication username, password, ok := r.BasicAuth() if !ok { log.Println(r.RemoteAddr, "unauthenticated", username) @@ -338,11 +453,20 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, "unauthenticated", http.StatusUnauthorized) return } + + // Form parsing var err error if err = r.ParseMultipartForm(1 << 20); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } + pkgNames, exists := r.MultipartForm.Value["name"] + if !exists || len(pkgNames) != 1 { + http.Error(w, "single name is expected in request", http.StatusBadRequest) + return + } + pkgName := normalizationRe.ReplaceAllString(pkgNames[0], "-") + dirPath := filepath.Join(*root, pkgName) var digestExpected []byte if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists { digestExpected, err = hex.DecodeString(digestExpectedHex[0]) @@ -352,41 +476,39 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { } } gpgSigsExpected := make(map[string]struct{}) + + // Checking is it internal package + if _, err = os.Stat(filepath.Join(dirPath, InternalFlag)); err != nil { + log.Println(r.RemoteAddr, "non-internal package", pkgName) + http.Error(w, "unknown internal package", http.StatusUnauthorized) + return + } + for _, file := range r.MultipartForm.File["content"] { filename := file.Filename gpgSigsExpected[filename+GPGSigExt] = struct{}{} log.Println(r.RemoteAddr, "put", filename, "by", username) - dir := filename[:strings.LastIndex(filename, "-")] - dirPath := filepath.Join(*root, dir) path := filepath.Join(dirPath, filename) if _, err = os.Stat(path); err == nil { log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "Already exists", http.StatusBadRequest) + http.Error(w, "already exists", http.StatusBadRequest) return } - if !mkdirForPkg(w, r, dir) { + if !mkdirForPkg(w, r, pkgName) { return } - internalPath := filepath.Join(dirPath, InternalFlag) - var dst *os.File - if _, err = os.Stat(internalPath); os.IsNotExist(err) { - if dst, err = os.Create(internalPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst.Close() - } src, err := file.Open() defer src.Close() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - dst, err = TempFile(dirPath) + dst, err := TempFile(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } + dstBuf := bufio.NewWriter(dst) hasher := sha256.New() wr := io.MultiWriter(hasher, dst) if _, err = io.Copy(wr, src); err != nil { @@ -395,6 +517,12 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + if err = dstBuf.Flush(); err != nil { + os.Remove(dst.Name()) + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } if err = dst.Sync(); err != nil { os.Remove(dst.Name()) dst.Close() @@ -417,7 +545,11 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if err = WriteFileSync(dirPath, path+SHA256Ext, digest); err != nil { + if err = DirSync(dirPath); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err = WriteFileSync(dirPath, path+"."+HashAlgoSHA256, digest); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -430,12 +562,10 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { } delete(gpgSigsExpected, filename) log.Println(r.RemoteAddr, "put", filename, "by", username) - dir := filename[:strings.LastIndex(filename, "-")] - dirPath := filepath.Join(*root, dir) path := filepath.Join(dirPath, filename) if _, err = os.Stat(path); err == nil { log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "Already exists", http.StatusBadRequest) + http.Error(w, "already exists", http.StatusBadRequest) return } src, err := file.Open() @@ -496,50 +626,6 @@ func handler(w http.ResponseWriter, r *http.Request) { } } -func goodIntegrity() bool { - dirs, err := ioutil.ReadDir(*root) - if err != nil { - log.Fatal(err) - } - hasher := sha256.New() - digest := make([]byte, sha256.Size) - isGood := true - var data []byte - var pkgName string - for _, dir := range dirs { - files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name())) - if err != nil { - log.Fatal(err) - } - for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { - continue - } - pkgName = strings.TrimSuffix(file.Name(), SHA256Ext) - data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName)) - if err != nil { - if os.IsNotExist(err) { - continue - } - log.Fatal(err) - } - hasher.Write(data) - data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name())) - if err != nil { - log.Fatal(err) - } - if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 { - log.Println(pkgName, "GOOD") - } else { - isGood = false - log.Println(pkgName, "BAD") - } - hasher.Reset() - } - } - return isGood -} - func main() { flag.Parse() if *warranty { @@ -563,6 +649,11 @@ func main() { if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") { log.Fatalln("Both -tls-cert and -tls-key are required") } + var err error + pypiURLParsed, err = url.Parse(*pypiURL) + if err != nil { + log.Fatalln(err) + } refreshPasswd() log.Println("root:", *root, "bind:", *bind) @@ -577,7 +668,9 @@ func main() { } http.HandleFunc(*norefreshURLPath, handler) http.HandleFunc(*refreshURLPath, handler) - http.HandleFunc(*gpgUpdateURLPath, handler) + if *gpgUpdateURLPath != "" { + http.HandleFunc(*gpgUpdateURLPath, handler) + } needsRefreshPasswd := make(chan os.Signal, 0) needsShutdown := make(chan os.Signal, 0)