X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=gocheese.go;h=d467bba22209ea413be5cc05f6a68d89c5e8d50f;hb=d9767ae738045ebf1bd483ae708af80fa384ca9f;hp=d6a1a70ba708b63911e02b5bfc55f5d79290aa84;hpb=11d218004e3a2668985a6d9b2628cb4b3fdc0051;p=gocheese.git diff --git a/gocheese.go b/gocheese.go index d6a1a70..d467bba 100644 --- a/gocheese.go +++ b/gocheese.go @@ -1,7 +1,7 @@ /* GoCheese -- Python private package repository and caching proxy -Copyright (C) 2019 Sergey Matveev - 2019 Elena Balakhonova +Copyright (C) 2019-2020 Sergey Matveev + 2019-2020 Elena Balakhonova This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -20,17 +20,11 @@ along with this program. If not, see . package main import ( - "bufio" "bytes" "context" - "crypto/md5" - "crypto/sha256" - "crypto/sha512" "encoding/hex" "flag" "fmt" - "hash" - "io" "io/ioutil" "log" "net" @@ -45,11 +39,11 @@ import ( "syscall" "time" - "golang.org/x/crypto/blake2b" "golang.org/x/net/netutil" ) const ( + Version = "2.5.0" HTMLBegin = ` @@ -75,15 +69,18 @@ You should have received a copy of the GNU General Public License along with this program. If not, see .` ) +const ( + HashAlgoSHA256 = "sha256" + HashAlgoBLAKE2b256 = "blake2_256" + HashAlgoSHA512 = "sha512" + HashAlgoMD5 = "md5" +) + var ( pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) normalizationRe = regexp.MustCompilePOSIX("[-_.]+") - HashAlgoSHA256 = "sha256" - HashAlgoBLAKE2b256 = "blake2_256" - HashAlgoSHA512 = "sha512" - HashAlgoMD5 = "md5" - knownHashAlgos []string = []string{ + knownHashAlgos []string = []string{ HashAlgoSHA256, HashAlgoBLAKE2b256, HashAlgoSHA512, @@ -99,246 +96,26 @@ var ( gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path") pypiURL = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL") passwdPath = flag.String("passwd", "passwd", "Path to file with authenticators") + logTimestamped = flag.Bool("log-timestamped", false, "Prepend timestmap to log messages") passwdCheck = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit") - fsck = flag.Bool("fsck", false, "Check integrity of all packages") + fsck = flag.Bool("fsck", false, "Check integrity of all packages (errors are in stderr)") maxClients = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") - Version string = "UNKNOWN" killed bool pypiURLParsed *url.URL ) -func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { - path := filepath.Join(*root, dir) +func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool { + path := filepath.Join(*root, pkgName) if _, err := os.Stat(path); os.IsNotExist(err) { if err = os.Mkdir(path, os.FileMode(0777)); err != nil { + log.Println("error", r.RemoteAddr, "mkdir", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "mkdir", dir) - } - return true -} - -func blake2b256New() hash.Hash { - h, err := blake2b.New256(nil) - if err != nil { - panic(err) - } - return h -} - -func refreshDir( - w http.ResponseWriter, - r *http.Request, - dir, - filenameGet string, - gpgUpdate bool, -) bool { - if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil { - return true - } - resp, err := http.Get(*pypiURL + dir + "/") - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - if !mkdirForPkg(w, r, dir) { - return false - } - dirPath := filepath.Join(*root, dir) - for _, lineRaw := range bytes.Split(body, []byte("\n")) { - submatches := pkgPyPI.FindStringSubmatch(string(lineRaw)) - if len(submatches) == 0 { - continue - } - uri := submatches[1] - filename := submatches[2] - pkgURL, err := url.Parse(uri) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - - if pkgURL.Fragment == "" { - log.Println(r.RemoteAddr, "pypi", filename, "no digest provided") - http.Error(w, "no digest provided", http.StatusBadGateway) - return false - } - digestInfo := strings.Split(pkgURL.Fragment, "=") - if len(digestInfo) == 1 { - // Ancient non PEP-0503 PyPIs, assume MD5 - digestInfo = []string{"md5", digestInfo[0]} - } else if len(digestInfo) != 2 { - log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided") - http.Error(w, "invalid digest provided", http.StatusBadGateway) - return false - } - digest, err := hex.DecodeString(digestInfo[1]) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - hashAlgo := digestInfo[0] - var hasherNew func() hash.Hash - var hashSize int - switch hashAlgo { - case HashAlgoMD5: - hasherNew = md5.New - hashSize = md5.Size - case HashAlgoSHA256: - hasherNew = sha256.New - hashSize = sha256.Size - case HashAlgoSHA512: - hasherNew = sha512.New - hashSize = sha512.Size - case HashAlgoBLAKE2b256: - hasherNew = blake2b256New - hashSize = blake2b.Size256 - default: - log.Println( - r.RemoteAddr, "pypi", filename, - "unknown digest algorithm", hashAlgo, - ) - http.Error(w, "unknown digest algorithm", http.StatusBadGateway) - return false - } - if len(digest) != hashSize { - log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length") - http.Error(w, "invalid digest length", http.StatusBadGateway) - return false - } - - pkgURL.Fragment = "" - if pkgURL.Host == "" { - uri = pypiURLParsed.ResolveReference(pkgURL).String() - } else { - uri = pkgURL.String() - } - - path := filepath.Join(dirPath, filename) - if filename == filenameGet { - if killed { - // Skip heavy remote call, when shutting down - http.Error(w, "shutting down", http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi download", filename) - resp, err = http.Get(uri) - if err != nil { - log.Println(r.RemoteAddr, "pypi download error:", err.Error()) - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - defer resp.Body.Close() - hasher := hasherNew() - hasherSHA256 := sha256.New() - dst, err := TempFile(dirPath) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - dstBuf := bufio.NewWriter(dst) - wrs := []io.Writer{hasher, dstBuf} - if hashAlgo != HashAlgoSHA256 { - wrs = append(wrs, hasherSHA256) - } - wr := io.MultiWriter(wrs...) - if _, err = io.Copy(wr, resp.Body); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = dstBuf.Flush(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if bytes.Compare(hasher.Sum(nil), digest) != 0 { - log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch") - os.Remove(dst.Name()) - dst.Close() - http.Error(w, "digest mismatch", http.StatusBadGateway) - return false - } - if err = dst.Sync(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = dst.Close(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = os.Rename(dst.Name(), path); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = DirSync(dirPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if hashAlgo != HashAlgoSHA256 { - hashAlgo = HashAlgoSHA256 - digest = hasherSHA256.Sum(nil) - for _, algo := range knownHashAlgos[1:] { - os.Remove(path + "." + algo) - } - } - } - if filename == filenameGet || gpgUpdate { - if _, err = os.Stat(path); err != nil { - goto GPGSigSkip - } - resp, err := http.Get(uri + GPGSigExt) - if err != nil { - goto GPGSigSkip - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - goto GPGSigSkip - } - sig, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - goto GPGSigSkip - } - if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) { - log.Println(r.RemoteAddr, "pypi non PGP signature", filename) - goto GPGSigSkip - } - if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi downloaded signature", filename) - } - GPGSigSkip: - path = path + "." + hashAlgo - _, err = os.Stat(path) - if err == nil { - continue - } - if !os.IsNotExist(err) { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi touch", filename) - if err = WriteFileSync(dirPath, path, digest); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } + log.Println(r.RemoteAddr, "mkdir", pkgName) } return true } @@ -346,6 +123,7 @@ func refreshDir( func listRoot(w http.ResponseWriter, r *http.Request) { files, err := ioutil.ReadDir(*root) if err != nil { + log.Println("error", r.RemoteAddr, "root", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -356,7 +134,7 @@ func listRoot(w http.ResponseWriter, r *http.Request) { result.WriteString(fmt.Sprintf( HTMLElement, *refreshURLPath+file.Name()+"/", - file.Name(), + "", file.Name(), )) } } @@ -367,20 +145,20 @@ func listRoot(w http.ResponseWriter, r *http.Request) { func listDir( w http.ResponseWriter, r *http.Request, - dir string, - autorefresh, - gpgUpdate bool, + pkgName string, + autorefresh, gpgUpdate bool, ) { - dirPath := filepath.Join(*root, dir) + dirPath := filepath.Join(*root, pkgName) if autorefresh { - if !refreshDir(w, r, dir, "", gpgUpdate) { + if !refreshDir(w, r, pkgName, "", gpgUpdate) { return } - } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) { + } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "", false) { return } fis, err := ioutil.ReadDir(dirPath) if err != nil { + log.Println("error", r.RemoteAddr, "list", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -389,7 +167,7 @@ func listDir( files[fi.Name()] = struct{}{} } var result bytes.Buffer - result.WriteString(fmt.Sprintf(HTMLBegin, dir)) + result.WriteString(fmt.Sprintf(HTMLBegin, pkgName)) for _, algo := range knownHashAlgos { for fn, _ := range files { if killed { @@ -403,6 +181,7 @@ func listDir( delete(files, fn) digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn)) if err != nil { + log.Println("error", r.RemoteAddr, "list", fn, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -416,7 +195,7 @@ func listDir( result.WriteString(fmt.Sprintf( HTMLElement, strings.Join([]string{ - *refreshURLPath, dir, "/", fnClean, + *refreshURLPath, pkgName, "/", fnClean, "#", algo, "=", hex.EncodeToString(digest), }, ""), gpgSigAttr, @@ -428,164 +207,17 @@ func listDir( w.Write(result.Bytes()) } -func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { +func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) { log.Println(r.RemoteAddr, "get", filename) - path := filepath.Join(*root, dir, filename) + path := filepath.Join(*root, pkgName, filename) if _, err := os.Stat(path); os.IsNotExist(err) { - if !refreshDir(w, r, dir, filename, false) { + if !refreshDir(w, r, pkgName, filename, false) { return } } http.ServeFile(w, r, path) } -func serveUpload(w http.ResponseWriter, r *http.Request) { - // Authentication - username, password, ok := r.BasicAuth() - if !ok { - log.Println(r.RemoteAddr, "unauthenticated", username) - http.Error(w, "unauthenticated", http.StatusUnauthorized) - return - } - auther, ok := passwords[username] - if !ok || !auther.Auth(password) { - log.Println(r.RemoteAddr, "unauthenticated", username) - http.Error(w, "unauthenticated", http.StatusUnauthorized) - return - } - - // Form parsing - var err error - if err = r.ParseMultipartForm(1 << 20); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - pkgNames, exists := r.MultipartForm.Value["name"] - if !exists || len(pkgNames) != 1 { - http.Error(w, "single name is expected in request", http.StatusBadRequest) - return - } - pkgName := normalizationRe.ReplaceAllString(pkgNames[0], "-") - dirPath := filepath.Join(*root, pkgName) - var digestExpected []byte - if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists { - digestExpected, err = hex.DecodeString(digestExpectedHex[0]) - if err != nil { - http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest) - return - } - } - gpgSigsExpected := make(map[string]struct{}) - - // Checking is it internal package - if _, err = os.Stat(filepath.Join(dirPath, InternalFlag)); err != nil { - log.Println(r.RemoteAddr, "non-internal package", pkgName) - http.Error(w, "unknown internal package", http.StatusUnauthorized) - return - } - - for _, file := range r.MultipartForm.File["content"] { - filename := file.Filename - gpgSigsExpected[filename+GPGSigExt] = struct{}{} - log.Println(r.RemoteAddr, "put", filename, "by", username) - path := filepath.Join(dirPath, filename) - if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "already exists", http.StatusBadRequest) - return - } - if !mkdirForPkg(w, r, pkgName) { - return - } - src, err := file.Open() - defer src.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst, err := TempFile(dirPath) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dstBuf := bufio.NewWriter(dst) - hasher := sha256.New() - wr := io.MultiWriter(hasher, dst) - if _, err = io.Copy(wr, src); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = dstBuf.Flush(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = dst.Sync(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst.Close() - digest := hasher.Sum(nil) - if digestExpected != nil { - if bytes.Compare(digestExpected, digest) == 0 { - log.Println(r.RemoteAddr, filename, "good checksum received") - } else { - log.Println(r.RemoteAddr, filename, "bad checksum received") - http.Error(w, "bad checksum", http.StatusBadRequest) - os.Remove(dst.Name()) - return - } - } - if err = os.Rename(dst.Name(), path); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = DirSync(dirPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = WriteFileSync(dirPath, path+"."+HashAlgoSHA256, digest); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - for _, file := range r.MultipartForm.File["gpg_signature"] { - filename := file.Filename - if _, exists := gpgSigsExpected[filename]; !exists { - http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest) - return - } - delete(gpgSigsExpected, filename) - log.Println(r.RemoteAddr, "put", filename, "by", username) - path := filepath.Join(dirPath, filename) - if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "already exists", http.StatusBadRequest) - return - } - src, err := file.Open() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - sig, err := ioutil.ReadAll(src) - src.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = WriteFileSync(dirPath, path, sig); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - func handler(w http.ResponseWriter, r *http.Request) { switch r.Method { case "GET": @@ -633,29 +265,39 @@ func main() { return } if *version { - fmt.Println("GoCheese version " + Version + " built with " + runtime.Version()) + fmt.Println("GoCheese", Version, "built with", runtime.Version()) return } + + if *logTimestamped { + log.SetFlags(log.Ldate | log.Lmicroseconds | log.Lshortfile) + } else { + log.SetFlags(log.Lshortfile) + } + log.SetOutput(os.Stdout) + if *fsck { if !goodIntegrity() { os.Exit(1) } return } + if *passwdCheck { refreshPasswd() return } + if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") { log.Fatalln("Both -tls-cert and -tls-key are required") } + var err error pypiURLParsed, err = url.Parse(*pypiURL) if err != nil { log.Fatalln(err) } refreshPasswd() - log.Println("root:", *root, "bind:", *bind) ln, err := net.Listen("tcp", *bind) if err != nil { @@ -679,19 +321,25 @@ func main() { signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT) go func() { for range needsRefreshPasswd { - log.Println("Refreshing passwords") + log.Println("refreshing passwords") refreshPasswd() } }() go func(s *http.Server) { <-needsShutdown killed = true - log.Println("Shutting down") + log.Println("shutting down") ctx, cancel := context.WithTimeout(context.TODO(), time.Minute) exitErr <- s.Shutdown(ctx) cancel() }(server) + log.Println( + "GoCheese", Version, "listens:", + "root:", *root, + "bind:", *bind, + "pypi:", *pypiURL, + ) if *tlsCert == "" { err = server.Serve(ln) } else {