X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=gocheese.go;h=ff76c870c3183a6b81e462ee131caa9ad679934f;hb=e396b31d6baa52055648be298dbb33ca91bdf33d;hp=7a36129746ab96d1254564d69385323639531d1d;hpb=902150e27fa1530823817eb3227f9100b27d9410;p=gocheese.git diff --git a/gocheese.go b/gocheese.go index 7a36129..ff76c87 100644 --- a/gocheese.go +++ b/gocheese.go @@ -20,12 +20,16 @@ along with this program. If not, see . package main import ( + "bufio" "bytes" "context" + "crypto/md5" "crypto/sha256" + "crypto/sha512" "encoding/hex" "flag" "fmt" + "hash" "io" "io/ioutil" "log" @@ -45,12 +49,18 @@ import ( ) const ( - HTMLBegin = "Links for %s

Links for %s

\n" - HTMLEnd = "" - HTMLElement = "%s
\n" - SHA256Prefix = "sha256=" - SHA256Ext = ".sha256" + HTMLBegin = ` + + + Links for %s + + +` + HTMLEnd = " \n\n" + HTMLElement = " %s
\n" InternalFlag = ".internal" + GPGSigExt = ".asc" + GPGSigAttr = " data-gpg-sig=true" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -75,6 +85,7 @@ var ( tlsKey = flag.String("tls-key", "", "Path to TLS X.509 private key") norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path") refreshURLPath = flag.String("refresh", "/simple/", "Auto-refreshing URL path") + gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path") pypiURL = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL") passwdPath = flag.String("passwd", "passwd", "Path to file with authenticators") passwdCheck = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit") @@ -82,6 +93,11 @@ var ( maxClients = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") + + killed bool + + pypiURLParsed *url.URL + normalizationRe *regexp.Regexp = regexp.MustCompilePOSIX("[-_.]+") ) func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { @@ -96,19 +112,23 @@ func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { return true } -func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool { +func refreshDir( + w http.ResponseWriter, + r *http.Request, + dir, + filenameGet string, + gpgUpdate bool, +) bool { if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil { - log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir) return true } - log.Println(r.RemoteAddr, "pypi refresh", dir) resp, err := http.Get(*pypiURL + dir + "/") if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false } - defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false @@ -116,6 +136,7 @@ func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) if !mkdirForPkg(w, r, dir) { return false } + dirPath := filepath.Join(*root, dir) var submatches []string var uri string var filename string @@ -130,41 +151,110 @@ func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) uri = submatches[1] filename = submatches[2] if pkgURL, err = url.Parse(uri); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusBadGateway) + return false + } + + if pkgURL.Fragment == "" { + log.Println(r.RemoteAddr, "pypi", filename, "no digest provided") + http.Error(w, "no digest provided", http.StatusBadGateway) return false } - digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix)) + digestInfo := strings.Split(pkgURL.Fragment, "=") + if len(digestInfo) == 1 { + // Ancient non PEP-0503 PyPIs, assume MD5 + digestInfo = []string{"md5", digestInfo[0]} + } else if len(digestInfo) != 2 { + log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided") + http.Error(w, "invalid digest provided", http.StatusBadGateway) + return false + } + digest, err = hex.DecodeString(digestInfo[1]) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false } + var hasherNew func() hash.Hash + var hashExt string + var hashSize int + switch digestInfo[0] { + case "md5": + hashExt = ".md5" + hasherNew = md5.New + hashSize = md5.Size + case "sha256": + hashExt = ".sha256" + hasherNew = sha256.New + hashSize = sha256.Size + case "sha512": + hashExt = ".sha512" + hasherNew = sha512.New + hashSize = sha512.Size + default: + log.Println( + r.RemoteAddr, "pypi", filename, + "unknown digest algorithm", digestInfo[0], + ) + http.Error(w, "unknown digest algorithm", http.StatusBadGateway) + return false + } + if len(digest) != hashSize { + log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length") + http.Error(w, "invalid digest length", http.StatusBadGateway) + return false + } + + pkgURL.Fragment = "" + if pkgURL.Host == "" { + uri = pypiURLParsed.ResolveReference(pkgURL).String() + } else { + uri = pkgURL.String() + } + path = filepath.Join(dirPath, filename) if filename == filenameGet { + if killed { + // Skip heavy remote call, when shutting down + http.Error(w, "shutting down", http.StatusInternalServerError) + return false + } log.Println(r.RemoteAddr, "pypi download", filename) - path = filepath.Join(*root, dir, filename) resp, err = http.Get(uri) if err != nil { + log.Println(r.RemoteAddr, "pypi download error:", err.Error()) http.Error(w, err.Error(), http.StatusBadGateway) return false } defer resp.Body.Close() - hasher := sha256.New() - dst, err := TempFile(filepath.Join(*root, dir)) + hasher := hasherNew() + hasherOur := sha256.New() + dst, err := TempFile(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } - wr := io.MultiWriter(hasher, dst) + dstBuf := bufio.NewWriter(dst) + wrs := []io.Writer{hasher, dstBuf} + if hashExt != ".sha256" { + wrs = append(wrs, hasherOur) + } + wr := io.MultiWriter(wrs...) if _, err = io.Copy(wr, resp.Body); err != nil { os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return false } + if err = dstBuf.Flush(); err != nil { + os.Remove(dst.Name()) + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } if bytes.Compare(hasher.Sum(nil), digest) != 0 { log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch") os.Remove(dst.Name()) dst.Close() - http.Error(w, err.Error(), http.StatusBadGateway) + http.Error(w, "digest mismatch", http.StatusBadGateway) return false } if err = dst.Sync(); err != nil { @@ -173,13 +263,48 @@ func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) http.Error(w, err.Error(), http.StatusInternalServerError) return false } - dst.Close() + if err = dst.Close(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } if err = os.Rename(dst.Name(), path); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } + if err = DirSync(dirPath); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } + if hashExt != ".sha256" { + hashExt = ".sha256" + digest = hasherOur.Sum(nil) + } + } + if filename == filenameGet || gpgUpdate { + if _, err = os.Stat(path); err != nil { + goto GPGSigSkip + } + resp, err := http.Get(uri + GPGSigExt) + if err != nil { + goto GPGSigSkip + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + goto GPGSigSkip + } + sig, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + goto GPGSigSkip + } + if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } + log.Println(r.RemoteAddr, "pypi downloaded signature", filename) } - path = filepath.Join(*root, dir, filename+SHA256Ext) + GPGSigSkip: + path = path + hashExt _, err = os.Stat(path) if err == nil { continue @@ -189,7 +314,7 @@ func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) return false } log.Println(r.RemoteAddr, "pypi touch", filename) - if err = ioutil.WriteFile(path, digest, os.FileMode(0666)); err != nil { + if err = WriteFileSync(dirPath, path, digest); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } @@ -198,70 +323,105 @@ func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) } func listRoot(w http.ResponseWriter, r *http.Request) { - log.Println(r.RemoteAddr, "root") files, err := ioutil.ReadDir(*root) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root"))) + var result bytes.Buffer + result.WriteString(fmt.Sprintf(HTMLBegin, "root")) for _, file := range files { if file.Mode().IsDir() { - w.Write([]byte(fmt.Sprintf( + result.WriteString(fmt.Sprintf( HTMLElement, *refreshURLPath+file.Name()+"/", file.Name(), - ))) + )) } } - w.Write([]byte(HTMLEnd)) + result.WriteString(HTMLEnd) + w.Write(result.Bytes()) } -func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) { - log.Println(r.RemoteAddr, "dir", dir) +func listDir( + w http.ResponseWriter, + r *http.Request, + dir string, + autorefresh, + gpgUpdate bool, +) { dirPath := filepath.Join(*root, dir) if autorefresh { - if !refreshDir(w, r, dir, "") { + if !refreshDir(w, r, dir, "", gpgUpdate) { return } - } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") { + } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) { return } - files, err := ioutil.ReadDir(dirPath) + fis, err := ioutil.ReadDir(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir))) - var data []byte - var filenameClean string - for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { - continue - } - data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name())) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + var result bytes.Buffer + result.WriteString(fmt.Sprintf(HTMLBegin, dir)) + var digest []byte + var gpgSigAttr string + var fnClean string + files := make(map[string]struct{}, len(fis)/2) + for _, fi := range fis { + files[fi.Name()] = struct{}{} + } + for _, algoExt := range []string{".sha256", ".sha512", ".md5"} { + for fn, _ := range files { + if killed { + // Skip expensive I/O when shutting down + http.Error(w, "shutting down", http.StatusInternalServerError) + return + } + if !strings.HasSuffix(fn, algoExt) { + continue + } + digest, err = ioutil.ReadFile(filepath.Join(dirPath, fn)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + fnClean = strings.TrimSuffix(fn, algoExt) + if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); os.IsNotExist(err) { + gpgSigAttr = "" + } else { + gpgSigAttr = GPGSigAttr + } + result.WriteString(fmt.Sprintf( + HTMLElement, + strings.Join([]string{ + *refreshURLPath, dir, "/", fnClean, + "#", algoExt[1:], "=", hex.EncodeToString(digest), + }, ""), + gpgSigAttr, + fnClean, + )) + for _, n := range []string{ + fnClean, + fnClean + GPGSigExt, + fnClean + ".sha256", + fnClean + ".sha512", + fnClean + ".md5", + } { + delete(files, n) + } } - filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext) - w.Write([]byte(fmt.Sprintf( - HTMLElement, - strings.Join([]string{ - *refreshURLPath, dir, "/", - filenameClean, "#", SHA256Prefix, string(data), - }, ""), - filenameClean, - ))) - } - w.Write([]byte(HTMLEnd)) + } + result.WriteString(HTMLEnd) + w.Write(result.Bytes()) } func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { - log.Println(r.RemoteAddr, "pkg", filename) + log.Println(r.RemoteAddr, "get", filename) path := filepath.Join(*root, dir, filename) if _, err := os.Stat(path); os.IsNotExist(err) { - if !refreshDir(w, r, dir, filename) { + if !refreshDir(w, r, dir, filename, false) { return } } @@ -269,6 +429,7 @@ func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { } func serveUpload(w http.ResponseWriter, r *http.Request) { + // Authentication username, password, ok := r.BasicAuth() if !ok { log.Println(r.RemoteAddr, "unauthenticated", username) @@ -281,45 +442,62 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, "unauthenticated", http.StatusUnauthorized) return } + + // Form parsing var err error if err = r.ParseMultipartForm(1 << 20); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } + pkgNames, exists := r.MultipartForm.Value["name"] + if !exists || len(pkgNames) != 1 { + http.Error(w, "single name is expected in request", http.StatusBadRequest) + return + } + dir := normalizationRe.ReplaceAllString(pkgNames[0], "-") + dirPath := filepath.Join(*root, dir) + var digestExpected []byte + if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists { + digestExpected, err = hex.DecodeString(digestExpectedHex[0]) + if err != nil { + http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest) + return + } + } + gpgSigsExpected := make(map[string]struct{}) + + // Checking is it internal package + if _, err = os.Stat(filepath.Join(dirPath, InternalFlag)); err != nil { + log.Println(r.RemoteAddr, "non-internal package", dir) + http.Error(w, "unknown internal package", http.StatusUnauthorized) + return + } + for _, file := range r.MultipartForm.File["content"] { filename := file.Filename - log.Println(r.RemoteAddr, "upload", filename, "by", username) - dir := filename[:strings.LastIndex(filename, "-")] - dirPath := filepath.Join(*root, dir) + gpgSigsExpected[filename+GPGSigExt] = struct{}{} + log.Println(r.RemoteAddr, "put", filename, "by", username) path := filepath.Join(dirPath, filename) if _, err = os.Stat(path); err == nil { log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "Already exists", http.StatusBadRequest) + http.Error(w, "already exists", http.StatusBadRequest) return } if !mkdirForPkg(w, r, dir) { return } - internalPath := filepath.Join(dirPath, InternalFlag) - var dst *os.File - if _, err = os.Stat(internalPath); os.IsNotExist(err) { - if dst, err = os.Create(internalPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst.Close() - } src, err := file.Open() defer src.Close() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - dst, err = TempFile(dirPath) + dst, err := TempFile(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } + dstBuf := bufio.NewWriter(dst) hasher := sha256.New() wr := io.MultiWriter(hasher, dst) if _, err = io.Copy(wr, src); err != nil { @@ -328,6 +506,12 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + if err = dstBuf.Flush(); err != nil { + os.Remove(dst.Name()) + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } if err = dst.Sync(); err != nil { os.Remove(dst.Name()) dst.Close() @@ -335,11 +519,56 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { return } dst.Close() + digest := hasher.Sum(nil) + if digestExpected != nil { + if bytes.Compare(digestExpected, digest) == 0 { + log.Println(r.RemoteAddr, filename, "good checksum received") + } else { + log.Println(r.RemoteAddr, filename, "bad checksum received") + http.Error(w, "bad checksum", http.StatusBadRequest) + os.Remove(dst.Name()) + return + } + } if err = os.Rename(dst.Name(), path); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if err = ioutil.WriteFile(path+SHA256Ext, hasher.Sum(nil), os.FileMode(0666)); err != nil { + if err = DirSync(dirPath); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err = WriteFileSync(dirPath, path+".sha256", digest); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + for _, file := range r.MultipartForm.File["gpg_signature"] { + filename := file.Filename + if _, exists := gpgSigsExpected[filename]; !exists { + http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest) + return + } + delete(gpgSigsExpected, filename) + log.Println(r.RemoteAddr, "put", filename, "by", username) + path := filepath.Join(dirPath, filename) + if _, err = os.Stat(path); err == nil { + log.Println(r.RemoteAddr, "already exists", filename) + http.Error(w, "already exists", http.StatusBadRequest) + return + } + src, err := file.Open() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + sig, err := ioutil.ReadAll(src) + src.Close() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err = WriteFileSync(dirPath, path, sig); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -347,15 +576,23 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { } func handler(w http.ResponseWriter, r *http.Request) { - if r.Method == "GET" { + switch r.Method { + case "GET": var path string var autorefresh bool + var gpgUpdate bool if strings.HasPrefix(r.URL.Path, *norefreshURLPath) { path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath) - autorefresh = false - } else { + } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) { path = strings.TrimPrefix(r.URL.Path, *refreshURLPath) autorefresh = true + } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) { + path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath) + autorefresh = true + gpgUpdate = true + } else { + http.Error(w, "unknown action", http.StatusBadRequest) + return } parts := strings.Split(strings.TrimSuffix(path, "/"), "/") if len(parts) > 2 { @@ -366,13 +603,15 @@ func handler(w http.ResponseWriter, r *http.Request) { if parts[0] == "" { listRoot(w, r) } else { - listDir(w, r, parts[0], autorefresh) + listDir(w, r, parts[0], autorefresh, gpgUpdate) } } else { servePkg(w, r, parts[0], parts[1]) } - } else if r.Method == "POST" { + case "POST": serveUpload(w, r) + default: + http.Error(w, "unknown action", http.StatusBadRequest) } } @@ -392,10 +631,10 @@ func goodIntegrity() bool { log.Fatal(err) } for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { + if !strings.HasSuffix(file.Name(), ".sha256") { continue } - pkgName = strings.TrimSuffix(file.Name(), SHA256Ext) + pkgName = strings.TrimSuffix(file.Name(), ".sha256") data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName)) if err != nil { if os.IsNotExist(err) { @@ -409,10 +648,10 @@ func goodIntegrity() bool { log.Fatal(err) } if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 { - log.Println(pkgName, "GOOD") + fmt.Println(pkgName, "GOOD") } else { isGood = false - log.Println(pkgName, "BAD") + fmt.Println(pkgName, "BAD") } hasher.Reset() } @@ -443,6 +682,11 @@ func main() { if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") { log.Fatalln("Both -tls-cert and -tls-key are required") } + var err error + pypiURLParsed, err = url.Parse(*pypiURL) + if err != nil { + log.Fatalln(err) + } refreshPasswd() log.Println("root:", *root, "bind:", *bind) @@ -457,10 +701,11 @@ func main() { } http.HandleFunc(*norefreshURLPath, handler) http.HandleFunc(*refreshURLPath, handler) + http.HandleFunc(*gpgUpdateURLPath, handler) needsRefreshPasswd := make(chan os.Signal, 0) needsShutdown := make(chan os.Signal, 0) - killed := make(chan error, 0) + exitErr := make(chan error, 0) signal.Notify(needsRefreshPasswd, syscall.SIGHUP) signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT) go func() { @@ -471,9 +716,10 @@ func main() { }() go func(s *http.Server) { <-needsShutdown + killed = true log.Println("Shutting down") ctx, cancel := context.WithTimeout(context.TODO(), time.Minute) - killed <- s.Shutdown(ctx) + exitErr <- s.Shutdown(ctx) cancel() }(server) @@ -485,7 +731,7 @@ func main() { if err != http.ErrServerClosed { log.Fatal(err) } - if err := <-killed; err != nil { + if err := <-exitErr; err != nil { log.Fatal(err) } }