X-Git-Url: http://www.git.cypherpunks.ru/?p=gocheese.git;a=blobdiff_plain;f=gocheese.go;h=c1f13f6b193a4ddfe185ef95a3bb7636836fddda;hp=9035e99ccca883e1422520f8c3ee580606c06ae3;hb=b036ee436eb9bd8889734232a22d3f24be5c9ee2;hpb=beb994417f69cf1dbb197e904477f4fbb39f2677 diff --git a/gocheese.go b/gocheese.go index 9035e99..c1f13f6 100644 --- a/gocheese.go +++ b/gocheese.go @@ -1,6 +1,7 @@ /* GoCheese -- Python private package repository and caching proxy Copyright (C) 2019 Sergey Matveev + 2019 Elena Balakhonova This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -20,29 +21,39 @@ package main import ( "bytes" - "crypto/sha256" + "context" "encoding/hex" "flag" "fmt" - "io" "io/ioutil" "log" + "net" "net/http" "net/url" "os" + "os/signal" "path/filepath" "regexp" "runtime" "strings" + "syscall" + "time" + + "golang.org/x/net/netutil" ) const ( - HTMLBegin = "Links for %s

Links for %s

\n" - HTMLEnd = "" - HTMLElement = "%s
\n" - SHA256Prefix = "sha256=" - SHA256Ext = ".sha256" + HTMLBegin = ` + + + Links for %s + + +` + HTMLEnd = " \n\n" + HTMLElement = " %s
\n" InternalFlag = ".internal" + GPGSigExt = ".asc" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -58,26 +69,44 @@ along with this program. If not, see .` ) var ( + pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) + normalizationRe = regexp.MustCompilePOSIX("[-_.]+") + + HashAlgoSHA256 = "sha256" + HashAlgoBLAKE2b256 = "blake2_256" + HashAlgoSHA512 = "sha512" + HashAlgoMD5 = "md5" + knownHashAlgos []string = []string{ + HashAlgoSHA256, + HashAlgoBLAKE2b256, + HashAlgoSHA512, + HashAlgoMD5, + } + root = flag.String("root", "./packages", "Path to packages directory") bind = flag.String("bind", "[::]:8080", "Address to bind to") + tlsCert = flag.String("tls-cert", "", "Path to TLS X.509 certificate") + tlsKey = flag.String("tls-key", "", "Path to TLS X.509 private key") norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path") refreshURLPath = flag.String("refresh", "/simple/", "Auto-refreshing URL path") + gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path") pypiURL = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL") - passwdPath = flag.String("passwd", "passwd", "Path to file with login:password lines") + passwdPath = flag.String("passwd", "passwd", "Path to file with authenticators") + passwdCheck = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit") fsck = flag.Bool("fsck", false, "Check integrity of all packages") + maxClients = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") - pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) - Version string = "UNKNOWN" - - passwords map[string]string = make(map[string]string) + Version string = "UNKNOWN" + killed bool + pypiURLParsed *url.URL ) func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { path := filepath.Join(*root, dir) if _, err := os.Stat(path); os.IsNotExist(err) { - if err = os.Mkdir(path, 0700); err != nil { + if err = os.Mkdir(path, os.FileMode(0777)); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } @@ -86,265 +115,120 @@ func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { return true } -func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool { - if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil { - log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir) - return true - } - log.Println(r.RemoteAddr, "pypi refresh", dir) - resp, err := http.Get(*pypiURL + dir + "/") - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - if !mkdirForPkg(w, r, dir) { - return false - } - var submatches []string - var uri string - var filename string - var path string - var pkgURL *url.URL - var digest []byte - for _, lineRaw := range bytes.Split(body, []byte("\n")) { - submatches = pkgPyPI.FindStringSubmatch(string(lineRaw)) - if len(submatches) == 0 { - continue - } - uri = submatches[1] - filename = submatches[2] - if pkgURL, err = url.Parse(uri); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix)) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - if filename == filenameGet { - log.Println(r.RemoteAddr, "pypi download", filename) - path = filepath.Join(*root, dir, filename) - resp, err = http.Get(uri) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - defer resp.Body.Close() - hasher := sha256.New() - dst, err := ioutil.TempFile(filepath.Join(*root, dir), "") - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - wr := io.MultiWriter(hasher, dst) - if _, err = io.Copy(wr, resp.Body); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if bytes.Compare(hasher.Sum(nil), digest) != 0 { - log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch") - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - if err = dst.Sync(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - dst.Close() - if err = os.Rename(dst.Name(), path); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - } - path = filepath.Join(*root, dir, filename+SHA256Ext) - _, err = os.Stat(path) - if err == nil { - continue - } else { - if !os.IsNotExist(err) { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - } - log.Println(r.RemoteAddr, "pypi touch", filename) - if err = ioutil.WriteFile(path, digest, os.FileMode(0600)); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - } - return true -} - func listRoot(w http.ResponseWriter, r *http.Request) { - log.Println(r.RemoteAddr, "root") files, err := ioutil.ReadDir(*root) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root"))) + var result bytes.Buffer + result.WriteString(fmt.Sprintf(HTMLBegin, "root")) for _, file := range files { if file.Mode().IsDir() { - w.Write([]byte(fmt.Sprintf( + result.WriteString(fmt.Sprintf( HTMLElement, *refreshURLPath+file.Name()+"/", file.Name(), - ))) + )) } } - w.Write([]byte(HTMLEnd)) + result.WriteString(HTMLEnd) + w.Write(result.Bytes()) } -func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) { - log.Println(r.RemoteAddr, "dir", dir) +func listDir( + w http.ResponseWriter, + r *http.Request, + dir string, + autorefresh, + gpgUpdate bool, +) { dirPath := filepath.Join(*root, dir) if autorefresh { - if !refreshDir(w, r, dir, "") { + if !refreshDir(w, r, dir, "", gpgUpdate) { return } - } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") { + } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) { return } - files, err := ioutil.ReadDir(dirPath) + fis, err := ioutil.ReadDir(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir))) - var data []byte - var filenameClean string - for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { - continue - } - data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name())) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + files := make(map[string]struct{}, len(fis)/2) + for _, fi := range fis { + files[fi.Name()] = struct{}{} + } + var result bytes.Buffer + result.WriteString(fmt.Sprintf(HTMLBegin, dir)) + for _, algo := range knownHashAlgos { + for fn, _ := range files { + if killed { + // Skip expensive I/O when shutting down + http.Error(w, "shutting down", http.StatusInternalServerError) + return + } + if !strings.HasSuffix(fn, "."+algo) { + continue + } + delete(files, fn) + digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + fnClean := strings.TrimSuffix(fn, "."+algo) + delete(files, fnClean) + gpgSigAttr := "" + if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil { + gpgSigAttr = " data-gpg-sig=true" + delete(files, fnClean+GPGSigExt) + } + result.WriteString(fmt.Sprintf( + HTMLElement, + strings.Join([]string{ + *refreshURLPath, dir, "/", fnClean, + "#", algo, "=", hex.EncodeToString(digest), + }, ""), + gpgSigAttr, + fnClean, + )) } - filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext) - w.Write([]byte(fmt.Sprintf( - HTMLElement, - strings.Join([]string{ - *refreshURLPath, dir, "/", - filenameClean, "#", SHA256Prefix, string(data), - }, ""), - filenameClean, - ))) } - w.Write([]byte(HTMLEnd)) + result.WriteString(HTMLEnd) + w.Write(result.Bytes()) } func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { - log.Println(r.RemoteAddr, "pkg", filename) + log.Println(r.RemoteAddr, "get", filename) path := filepath.Join(*root, dir, filename) if _, err := os.Stat(path); os.IsNotExist(err) { - if !refreshDir(w, r, dir, filename) { + if !refreshDir(w, r, dir, filename, false) { return } } http.ServeFile(w, r, path) } -func serveUpload(w http.ResponseWriter, r *http.Request) { - username, password, ok := r.BasicAuth() - if !ok || passwords[username] != password { - log.Println(r.RemoteAddr, "unauthenticated", username) - http.Error(w, "unauthenticated", http.StatusUnauthorized) - return - } - var err error - if err = r.ParseMultipartForm(1 << 20); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - for _, file := range r.MultipartForm.File["content"] { - filename := file.Filename - log.Println(r.RemoteAddr, "upload", filename, "by", username) - dir := filename[:strings.LastIndex(filename, "-")] - dirPath := filepath.Join(*root, dir) - path := filepath.Join(dirPath, filename) - if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "Already exists", http.StatusBadRequest) - return - } - if !mkdirForPkg(w, r, dir) { - return - } - internalPath := filepath.Join(dirPath, InternalFlag) - var dst *os.File - if _, err = os.Stat(internalPath); os.IsNotExist(err) { - if dst, err = os.Create(internalPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst.Close() - } - src, err := file.Open() - defer src.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst, err = ioutil.TempFile(dirPath, "") - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - hasher := sha256.New() - wr := io.MultiWriter(hasher, dst) - if _, err = io.Copy(wr, src); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = dst.Sync(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst.Close() - if err = os.Rename(dst.Name(), path); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = ioutil.WriteFile( - path+SHA256Ext, - hasher.Sum(nil), - os.FileMode(0600), - ); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - func handler(w http.ResponseWriter, r *http.Request) { - if r.Method == "GET" { + switch r.Method { + case "GET": var path string var autorefresh bool + var gpgUpdate bool if strings.HasPrefix(r.URL.Path, *norefreshURLPath) { path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath) - autorefresh = false - } else { + } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) { path = strings.TrimPrefix(r.URL.Path, *refreshURLPath) autorefresh = true + } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) { + path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath) + autorefresh = true + gpgUpdate = true + } else { + http.Error(w, "unknown action", http.StatusBadRequest) + return } parts := strings.Split(strings.TrimSuffix(path, "/"), "/") if len(parts) > 2 { @@ -355,60 +239,18 @@ func handler(w http.ResponseWriter, r *http.Request) { if parts[0] == "" { listRoot(w, r) } else { - listDir(w, r, parts[0], autorefresh) + listDir(w, r, parts[0], autorefresh, gpgUpdate) } } else { servePkg(w, r, parts[0], parts[1]) } - } else if r.Method == "POST" { + case "POST": serveUpload(w, r) + default: + http.Error(w, "unknown action", http.StatusBadRequest) } } -func goodIntegrity() bool { - dirs, err := ioutil.ReadDir(*root) - if err != nil { - log.Fatal(err) - } - hasher := sha256.New() - digest := make([]byte, sha256.Size) - isGood := true - var data []byte - var pkgName string - for _, dir := range dirs { - files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name())) - if err != nil { - log.Fatal(err) - } - for _, file := range files { - if !strings.HasSuffix(file.Name(), SHA256Ext) { - continue - } - pkgName = strings.TrimSuffix(file.Name(), SHA256Ext) - data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName)) - if err != nil { - if os.IsNotExist(err) { - continue - } - log.Fatal(err) - } - hasher.Write(data) - data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name())) - if err != nil { - log.Fatal(err) - } - if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 { - log.Println(pkgName, "GOOD") - } else { - isGood = false - log.Println(pkgName, "BAD") - } - hasher.Reset() - } - } - return isGood -} - func main() { flag.Parse() if *warranty { @@ -425,19 +267,65 @@ func main() { } return } - passwd, err := ioutil.ReadFile(*passwdPath) + if *passwdCheck { + refreshPasswd() + return + } + if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") { + log.Fatalln("Both -tls-cert and -tls-key are required") + } + var err error + pypiURLParsed, err = url.Parse(*pypiURL) + if err != nil { + log.Fatalln(err) + } + refreshPasswd() + log.Println("root:", *root, "bind:", *bind) + + ln, err := net.Listen("tcp", *bind) if err != nil { log.Fatal(err) } - for _, credentials := range strings.Split(strings.TrimRight(string(passwd), "\n"), "\n") { - splitted := strings.Split(credentials, ":") - if len(splitted) != 2 { - log.Fatal("Wrong login:password format") - } - passwords[splitted[0]] = splitted[1] + ln = netutil.LimitListener(ln, *maxClients) + server := &http.Server{ + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, } - log.Println("root:", *root, "bind:", *bind) http.HandleFunc(*norefreshURLPath, handler) http.HandleFunc(*refreshURLPath, handler) - log.Fatal(http.ListenAndServe(*bind, nil)) + if *gpgUpdateURLPath != "" { + http.HandleFunc(*gpgUpdateURLPath, handler) + } + + needsRefreshPasswd := make(chan os.Signal, 0) + needsShutdown := make(chan os.Signal, 0) + exitErr := make(chan error, 0) + signal.Notify(needsRefreshPasswd, syscall.SIGHUP) + signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT) + go func() { + for range needsRefreshPasswd { + log.Println("Refreshing passwords") + refreshPasswd() + } + }() + go func(s *http.Server) { + <-needsShutdown + killed = true + log.Println("Shutting down") + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute) + exitErr <- s.Shutdown(ctx) + cancel() + }(server) + + if *tlsCert == "" { + err = server.Serve(ln) + } else { + err = server.ServeTLS(ln, *tlsCert, *tlsKey) + } + if err != http.ErrServerClosed { + log.Fatal(err) + } + if err := <-exitErr; err != nil { + log.Fatal(err) + } }