X-Git-Url: http://www.git.cypherpunks.ru/?p=gocheese.git;a=blobdiff_plain;f=gocheese.go;h=63854169f549e0e301fd72a2af488b74e77196a7;hp=655908d47f789735bc84033a9ef6b006991d0185;hb=0f9f81e55a5db0fa50b539554b078ae4ffdb29bd;hpb=b2c61aa43c08395b0d38fd6def46f369e0366e9b diff --git a/gocheese.go b/gocheese.go index 655908d..6385416 100644 --- a/gocheese.go +++ b/gocheese.go @@ -1,7 +1,7 @@ /* GoCheese -- Python private package repository and caching proxy -Copyright (C) 2019 Sergey Matveev - 2019 Elena Balakhonova +Copyright (C) 2019-2021 Sergey Matveev + 2019-2021 Elena Balakhonova This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -23,10 +23,11 @@ import ( "bytes" "context" "crypto/sha256" + "crypto/tls" "encoding/hex" + "errors" "flag" "fmt" - "io" "io/ioutil" "log" "net" @@ -45,6 +46,7 @@ import ( ) const ( + Version = "2.6.0" HTMLBegin = ` @@ -54,11 +56,8 @@ const ( ` HTMLEnd = " \n\n" HTMLElement = " %s
\n" - SHA256Prefix = "sha256=" - SHA256Ext = ".sha256" InternalFlag = ".internal" GPGSigExt = ".asc" - GPGSigAttr = " data-gpg-sig=true" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -73,9 +72,23 @@ You should have received a copy of the GNU General Public License along with this program. If not, see .` ) +const ( + HashAlgoSHA256 = "sha256" + HashAlgoBLAKE2b256 = "blake2_256" + HashAlgoSHA512 = "sha512" + HashAlgoMD5 = "md5" +) + var ( - pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) - Version string = "UNKNOWN" + pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) + normalizationRe = regexp.MustCompilePOSIX("[-_.]+") + + knownHashAlgos []string = []string{ + HashAlgoSHA256, + HashAlgoBLAKE2b256, + HashAlgoSHA512, + HashAlgoMD5, + } root = flag.String("root", "./packages", "Path to packages directory") bind = flag.String("bind", "[::]:8080", "Address to bind to") @@ -85,171 +98,28 @@ var ( refreshURLPath = flag.String("refresh", "/simple/", "Auto-refreshing URL path") gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path") pypiURL = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL") + pypiCertHash = flag.String("pypi-cert-hash", "", "Authenticate PyPI by its X.509 certificate's SHA256 hash") passwdPath = flag.String("passwd", "passwd", "Path to file with authenticators") + logTimestamped = flag.Bool("log-timestamped", false, "Prepend timestmap to log messages") passwdCheck = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit") - fsck = flag.Bool("fsck", false, "Check integrity of all packages") + fsck = flag.Bool("fsck", false, "Check integrity of all packages (errors are in stderr)") maxClients = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") - killed bool - - normalizationRe *regexp.Regexp = regexp.MustCompilePOSIX("[-_.]+") + killed bool + pypiURLParsed *url.URL ) -func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { - path := filepath.Join(*root, dir) +func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool { + path := filepath.Join(*root, pkgName) if _, err := os.Stat(path); os.IsNotExist(err) { if err = os.Mkdir(path, os.FileMode(0777)); err != nil { + log.Println("error", r.RemoteAddr, "mkdir", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "mkdir", dir) - } - return true -} - -func refreshDir( - w http.ResponseWriter, - r *http.Request, - dir, - filenameGet string, - gpgUpdate bool, -) bool { - if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil { - return true - } - resp, err := http.Get(*pypiURL + dir + "/") - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - if !mkdirForPkg(w, r, dir) { - return false - } - dirPath := filepath.Join(*root, dir) - var submatches []string - var uri string - var filename string - var path string - var pkgURL *url.URL - var digest []byte - for _, lineRaw := range bytes.Split(body, []byte("\n")) { - submatches = pkgPyPI.FindStringSubmatch(string(lineRaw)) - if len(submatches) == 0 { - continue - } - uri = submatches[1] - filename = submatches[2] - if pkgURL, err = url.Parse(uri); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if !strings.HasPrefix(pkgURL.Fragment, SHA256Prefix) { - log.Println(r.RemoteAddr, "pypi", filename, "no SHA256 digest provided") - http.Error(w, "no SHA256 digest provided", http.StatusBadGateway) - return false - } - digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix)) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - pkgURL.Fragment = "" - path = filepath.Join(dirPath, filename) - if filename == filenameGet { - if killed { - // Skip heavy remote call, when shutting down - http.Error(w, "shutting down", http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi download", filename) - resp, err = http.Get(pkgURL.String()) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - defer resp.Body.Close() - hasher := sha256.New() - dst, err := TempFile(dirPath) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - wr := io.MultiWriter(hasher, dst) - if _, err = io.Copy(wr, resp.Body); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if bytes.Compare(hasher.Sum(nil), digest) != 0 { - log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch") - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - if err = dst.Sync(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - dst.Close() - if err = os.Rename(dst.Name(), path); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = DirSync(dirPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - } - if filename == filenameGet || gpgUpdate { - if _, err = os.Stat(path); err != nil { - goto GPGSigSkip - } - resp, err := http.Get(pkgURL.String() + GPGSigExt) - if err != nil { - goto GPGSigSkip - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - goto GPGSigSkip - } - sig, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - goto GPGSigSkip - } - if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi downloaded signature", filename) - } - GPGSigSkip: - path = path + SHA256Ext - _, err = os.Stat(path) - if err == nil { - continue - } - if !os.IsNotExist(err) { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi touch", filename) - if err = WriteFileSync(dirPath, path, digest); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } + log.Println(r.RemoteAddr, "mkdir", pkgName) } return true } @@ -257,6 +127,7 @@ func refreshDir( func listRoot(w http.ResponseWriter, r *http.Request) { files, err := ioutil.ReadDir(*root) if err != nil { + log.Println("error", r.RemoteAddr, "root", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -267,7 +138,7 @@ func listRoot(w http.ResponseWriter, r *http.Request) { result.WriteString(fmt.Sprintf( HTMLElement, *refreshURLPath+file.Name()+"/", - file.Name(), + "", file.Name(), )) } } @@ -278,213 +149,79 @@ func listRoot(w http.ResponseWriter, r *http.Request) { func listDir( w http.ResponseWriter, r *http.Request, - dir string, - autorefresh, - gpgUpdate bool, + pkgName string, + autorefresh, gpgUpdate bool, ) { - dirPath := filepath.Join(*root, dir) + dirPath := filepath.Join(*root, pkgName) if autorefresh { - if !refreshDir(w, r, dir, "", gpgUpdate) { + if !refreshDir(w, r, pkgName, "", gpgUpdate) { return } - } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) { + } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "", false) { return } - files, err := ioutil.ReadDir(dirPath) + fis, err := ioutil.ReadDir(dirPath) if err != nil { + log.Println("error", r.RemoteAddr, "list", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } + files := make(map[string]struct{}, len(fis)/2) + for _, fi := range fis { + files[fi.Name()] = struct{}{} + } var result bytes.Buffer - result.WriteString(fmt.Sprintf(HTMLBegin, dir)) - var data []byte - var gpgSigAttr string - var filenameClean string - for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { - continue - } - if killed { - // Skip expensive I/O when shutting down - http.Error(w, "shutting down", http.StatusInternalServerError) - return - } - data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name())) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext) - if _, err = os.Stat(filepath.Join(dirPath, filenameClean+GPGSigExt)); os.IsNotExist(err) { - gpgSigAttr = "" - } else { - gpgSigAttr = GPGSigAttr + result.WriteString(fmt.Sprintf(HTMLBegin, pkgName)) + for _, algo := range knownHashAlgos { + for fn := range files { + if killed { + // Skip expensive I/O when shutting down + http.Error(w, "shutting down", http.StatusInternalServerError) + return + } + if !strings.HasSuffix(fn, "."+algo) { + continue + } + delete(files, fn) + digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn)) + if err != nil { + log.Println("error", r.RemoteAddr, "list", fn, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + fnClean := strings.TrimSuffix(fn, "."+algo) + delete(files, fnClean) + gpgSigAttr := "" + if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil { + gpgSigAttr = " data-gpg-sig=true" + delete(files, fnClean+GPGSigExt) + } + result.WriteString(fmt.Sprintf( + HTMLElement, + strings.Join([]string{ + *refreshURLPath, pkgName, "/", fnClean, + "#", algo, "=", hex.EncodeToString(digest), + }, ""), + gpgSigAttr, + fnClean, + )) } - result.WriteString(fmt.Sprintf( - HTMLElement, - strings.Join([]string{ - *refreshURLPath, dir, "/", - filenameClean, "#", SHA256Prefix, hex.EncodeToString(data), - }, ""), - gpgSigAttr, - filenameClean, - )) } result.WriteString(HTMLEnd) w.Write(result.Bytes()) } -func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { +func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) { log.Println(r.RemoteAddr, "get", filename) - path := filepath.Join(*root, dir, filename) + path := filepath.Join(*root, pkgName, filename) if _, err := os.Stat(path); os.IsNotExist(err) { - if !refreshDir(w, r, dir, filename, false) { + if !refreshDir(w, r, pkgName, filename, false) { return } } http.ServeFile(w, r, path) } -func serveUpload(w http.ResponseWriter, r *http.Request) { - // Authentication - username, password, ok := r.BasicAuth() - if !ok { - log.Println(r.RemoteAddr, "unauthenticated", username) - http.Error(w, "unauthenticated", http.StatusUnauthorized) - return - } - auther, ok := passwords[username] - if !ok || !auther.Auth(password) { - log.Println(r.RemoteAddr, "unauthenticated", username) - http.Error(w, "unauthenticated", http.StatusUnauthorized) - return - } - - // Form parsing - var err error - if err = r.ParseMultipartForm(1 << 20); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - pkgNames, exists := r.MultipartForm.Value["name"] - if !exists || len(pkgNames) != 1 { - http.Error(w, "single name is expected in request", http.StatusBadRequest) - return - } - dir := normalizationRe.ReplaceAllString(pkgNames[0], "-") - dirPath := filepath.Join(*root, dir) - var digestExpected []byte - if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists { - digestExpected, err = hex.DecodeString(digestExpectedHex[0]) - if err != nil { - http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest) - return - } - } - gpgSigsExpected := make(map[string]struct{}) - - // Checking is it internal package - if _, err = os.Stat(filepath.Join(dirPath, InternalFlag)); err != nil { - log.Println(r.RemoteAddr, "non-internal package", dir) - http.Error(w, "unknown internal package", http.StatusUnauthorized) - return - } - - for _, file := range r.MultipartForm.File["content"] { - filename := file.Filename - gpgSigsExpected[filename+GPGSigExt] = struct{}{} - log.Println(r.RemoteAddr, "put", filename, "by", username) - path := filepath.Join(dirPath, filename) - if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "already exists", http.StatusBadRequest) - return - } - if !mkdirForPkg(w, r, dir) { - return - } - src, err := file.Open() - defer src.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst, err := TempFile(dirPath) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - hasher := sha256.New() - wr := io.MultiWriter(hasher, dst) - if _, err = io.Copy(wr, src); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = dst.Sync(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst.Close() - digest := hasher.Sum(nil) - if digestExpected != nil { - if bytes.Compare(digestExpected, digest) == 0 { - log.Println(r.RemoteAddr, filename, "good checksum received") - } else { - log.Println(r.RemoteAddr, filename, "bad checksum received") - http.Error(w, "bad checksum", http.StatusBadRequest) - os.Remove(dst.Name()) - return - } - } - if err = os.Rename(dst.Name(), path); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = DirSync(dirPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = WriteFileSync(dirPath, path+SHA256Ext, digest); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - for _, file := range r.MultipartForm.File["gpg_signature"] { - filename := file.Filename - if _, exists := gpgSigsExpected[filename]; !exists { - http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest) - return - } - delete(gpgSigsExpected, filename) - log.Println(r.RemoteAddr, "put", filename, "by", username) - path := filepath.Join(dirPath, filename) - if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "already exists", http.StatusBadRequest) - return - } - src, err := file.Open() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - sig, err := ioutil.ReadAll(src) - src.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = WriteFileSync(dirPath, path, sig); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - func handler(w http.ResponseWriter, r *http.Request) { switch r.Method { case "GET": @@ -525,50 +262,6 @@ func handler(w http.ResponseWriter, r *http.Request) { } } -func goodIntegrity() bool { - dirs, err := ioutil.ReadDir(*root) - if err != nil { - log.Fatal(err) - } - hasher := sha256.New() - digest := make([]byte, sha256.Size) - isGood := true - var data []byte - var pkgName string - for _, dir := range dirs { - files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name())) - if err != nil { - log.Fatal(err) - } - for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { - continue - } - pkgName = strings.TrimSuffix(file.Name(), SHA256Ext) - data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName)) - if err != nil { - if os.IsNotExist(err) { - continue - } - log.Fatal(err) - } - hasher.Write(data) - data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name())) - if err != nil { - log.Fatal(err) - } - if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 { - fmt.Println(pkgName, "GOOD") - } else { - isGood = false - fmt.Println(pkgName, "BAD") - } - hasher.Reset() - } - } - return isGood -} - func main() { flag.Parse() if *warranty { @@ -576,24 +269,58 @@ func main() { return } if *version { - fmt.Println("GoCheese version " + Version + " built with " + runtime.Version()) + fmt.Println("GoCheese", Version, "built with", runtime.Version()) return } + + if *logTimestamped { + log.SetFlags(log.Ldate | log.Lmicroseconds | log.Lshortfile) + } else { + log.SetFlags(log.Lshortfile) + } + log.SetOutput(os.Stdout) + if *fsck { if !goodIntegrity() { os.Exit(1) } return } + if *passwdCheck { refreshPasswd() return } + if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") { log.Fatalln("Both -tls-cert and -tls-key are required") } + + var err error + pypiURLParsed, err = url.Parse(*pypiURL) + if err != nil { + log.Fatalln(err) + } refreshPasswd() - log.Println("root:", *root, "bind:", *bind) + if *pypiCertHash == "" { + pypiHTTPTransport = http.Transport{} + } else { + ourDgst, err := hex.DecodeString(*pypiCertHash) + if err != nil { + log.Fatalln(err) + } + pypiHTTPTransport = http.Transport{ + TLSClientConfig: &tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + spki := s.VerifiedChains[0][0].RawSubjectPublicKeyInfo + theirDgst := sha256.Sum256(spki) + if bytes.Compare(ourDgst, theirDgst[:]) != 0 { + return errors.New("certificate's digest mismatch") + } + return nil + }}, + } + } ln, err := net.Listen("tcp", *bind) if err != nil { @@ -606,7 +333,9 @@ func main() { } http.HandleFunc(*norefreshURLPath, handler) http.HandleFunc(*refreshURLPath, handler) - http.HandleFunc(*gpgUpdateURLPath, handler) + if *gpgUpdateURLPath != "" { + http.HandleFunc(*gpgUpdateURLPath, handler) + } needsRefreshPasswd := make(chan os.Signal, 0) needsShutdown := make(chan os.Signal, 0) @@ -615,19 +344,25 @@ func main() { signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT) go func() { for range needsRefreshPasswd { - log.Println("Refreshing passwords") + log.Println("refreshing passwords") refreshPasswd() } }() go func(s *http.Server) { <-needsShutdown killed = true - log.Println("Shutting down") + log.Println("shutting down") ctx, cancel := context.WithTimeout(context.TODO(), time.Minute) exitErr <- s.Shutdown(ctx) cancel() }(server) + log.Println( + "GoCheese", Version, "listens:", + "root:", *root, + "bind:", *bind, + "pypi:", *pypiURL, + ) if *tlsCert == "" { err = server.Serve(ln) } else {