/* GoCheese -- Python private package repository and caching proxy Copyright (C) 2019 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ // Python private package repository and caching proxy package main import ( "bytes" "crypto/sha256" "encoding/hex" "flag" "fmt" "io" "io/ioutil" "log" "net/http" "net/url" "os" "path/filepath" "regexp" "runtime" "strings" ) const ( HTMLBegin = "Links for %s

Links for %s

\n" HTMLEnd = "" HTMLElement = "%s
\n" SHA256Prefix = "sha256=" SHA256Ext = ".sha256" InternalFlag = ".internal" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see .` ) var ( root = flag.String("root", "./packages", "Path to packages directory") bind = flag.String("bind", "[::]:8080", "Address to bind to") simpleURLPath = flag.String("simple", "/simple/", "/simple/ URL path") refreshURLPath = flag.String("refresh", "/refresh/", "Auto-refreshing URL path") pypiURL = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL") auth = flag.String("auth", "spam:foo", "login:password,...") fsck = flag.Bool("fsck", false, "Check integrity of all packages") version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) Version string = "UNKNOWN" passwords map[string]string = make(map[string]string) ) func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { path := filepath.Join(*root, dir) if _, err := os.Stat(path); os.IsNotExist(err) { if err = os.Mkdir(path, 0700); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } log.Println(r.RemoteAddr, "mkdir", dir) } return true } func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool { if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil { log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir) return true } log.Println(r.RemoteAddr, "pypi refresh", dir) resp, err := http.Get(*pypiURL + dir + "/") if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false } if !mkdirForPkg(w, r, dir) { return false } var submatches []string var uri string var filename string var path string var pkgURL *url.URL var digest []byte for _, lineRaw := range bytes.Split(body, []byte("\n")) { submatches = pkgPyPI.FindStringSubmatch(string(lineRaw)) if len(submatches) == 0 { continue } uri = submatches[1] filename = submatches[2] if pkgURL, err = url.Parse(uri); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix)) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false } if filename == filenameGet { log.Println(r.RemoteAddr, "pypi download", filename) path = filepath.Join(*root, dir, filename) resp, err = http.Get(uri) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return false } defer resp.Body.Close() hasher := sha256.New() dst, err := ioutil.TempFile(filepath.Join(*root, dir), "") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } wr := io.MultiWriter(hasher, dst) if _, err = io.Copy(wr, resp.Body); err != nil { os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return false } if bytes.Compare(hasher.Sum(nil), digest) != 0 { log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch") os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusBadGateway) return false } if err = dst.Sync(); err != nil { os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return false } dst.Close() if err = os.Rename(dst.Name(), path); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } } path = filepath.Join(*root, dir, filename+SHA256Ext) _, err = os.Stat(path) if err == nil { continue } else { if !os.IsNotExist(err) { http.Error(w, err.Error(), http.StatusInternalServerError) return false } } log.Println(r.RemoteAddr, "pypi touch", filename) if err = ioutil.WriteFile(path, digest, os.FileMode(0600)); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return false } } return true } func listRoot(w http.ResponseWriter, r *http.Request) { log.Println(r.RemoteAddr, "root") files, err := ioutil.ReadDir(*root) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root"))) for _, file := range files { if file.Mode().IsDir() { w.Write([]byte(fmt.Sprintf( HTMLElement, *simpleURLPath+file.Name()+"/", file.Name(), ))) } } w.Write([]byte(HTMLEnd)) } func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) { log.Println(r.RemoteAddr, "dir", dir) dirPath := filepath.Join(*root, dir) if autorefresh { if !refreshDir(w, r, dir, "") { return } } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") { return } files, err := ioutil.ReadDir(dirPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir))) var data []byte var filenameClean string for _, file := range files { if !strings.HasSuffix(file.Name(), SHA256Ext) { continue } data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name())) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext) w.Write([]byte(fmt.Sprintf( HTMLElement, strings.Join([]string{ *simpleURLPath, dir, "/", filenameClean, "#", SHA256Prefix, string(data), }, ""), filenameClean, ))) } w.Write([]byte(HTMLEnd)) } func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { log.Println(r.RemoteAddr, "pkg", filename) path := filepath.Join(*root, dir, filename) if _, err := os.Stat(path); os.IsNotExist(err) { if !refreshDir(w, r, dir, filename) { return } } http.ServeFile(w, r, path) } func serveUpload(w http.ResponseWriter, r *http.Request) { username, password, ok := r.BasicAuth() if !ok || passwords[username] != password { log.Println(r.RemoteAddr, "unauthenticated", username) http.Error(w, "unauthenticated", http.StatusUnauthorized) return } var err error if err = r.ParseMultipartForm(1 << 20); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } for _, file := range r.MultipartForm.File["content"] { filename := file.Filename log.Println(r.RemoteAddr, "upload", filename, "by", username) dir := filename[:strings.LastIndex(filename, "-")] dirPath := filepath.Join(*root, dir) path := filepath.Join(dirPath, filename) if _, err = os.Stat(path); err == nil { log.Println(r.RemoteAddr, "already exists", filename) http.Error(w, "Already exists", http.StatusBadRequest) return } if !mkdirForPkg(w, r, dir) { return } internalPath := filepath.Join(dirPath, InternalFlag) var dst *os.File if _, err = os.Stat(internalPath); os.IsNotExist(err) { if dst, err = os.Create(internalPath); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } dst.Close() } src, err := file.Open() defer src.Close() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } dst, err = ioutil.TempFile(dirPath, "") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } hasher := sha256.New() wr := io.MultiWriter(hasher, dst) if _, err = io.Copy(wr, src); err != nil { os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return } if err = dst.Sync(); err != nil { os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return } dst.Close() if err = os.Rename(dst.Name(), path); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } if err = ioutil.WriteFile( path+SHA256Ext, hasher.Sum(nil), os.FileMode(0600), ); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } } } func handler(w http.ResponseWriter, r *http.Request) { if r.Method == "GET" { var path string var autorefresh bool if strings.HasPrefix(r.URL.Path, *simpleURLPath) { path = strings.TrimPrefix(r.URL.Path, *simpleURLPath) autorefresh = false } else { path = strings.TrimPrefix(r.URL.Path, *refreshURLPath) autorefresh = true } parts := strings.Split(strings.TrimSuffix(path, "/"), "/") if len(parts) > 2 { http.Error(w, "invalid path", http.StatusBadRequest) return } if len(parts) == 1 { if parts[0] == "" { listRoot(w, r) } else { listDir(w, r, parts[0], autorefresh) } } else { servePkg(w, r, parts[0], parts[1]) } } else if r.Method == "POST" { serveUpload(w, r) } } func goodIntegrity() bool { dirs, err := ioutil.ReadDir(*root) if err != nil { log.Fatal(err) } hasher := sha256.New() digest := make([]byte, sha256.Size) isGood := true var data []byte var pkgName string for _, dir := range dirs { files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name())) if err != nil { log.Fatal(err) } for _, file := range files { if !strings.HasSuffix(file.Name(), SHA256Ext) { continue } pkgName = strings.TrimSuffix(file.Name(), SHA256Ext) data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName)) if err != nil { if os.IsNotExist(err) { continue } log.Fatal(err) } hasher.Write(data) data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name())) if err != nil { log.Fatal(err) } if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 { log.Println(pkgName, "GOOD") } else { isGood = false log.Println(pkgName, "BAD") } hasher.Reset() } } return isGood } func main() { flag.Parse() if *warranty { fmt.Println(Warranty) return } if *version { fmt.Println("GoCheese version " + Version + " built with " + runtime.Version()) return } if *fsck { if !goodIntegrity() { os.Exit(1) } return } for _, credentials := range strings.Split(*auth, ",") { splitted := strings.Split(credentials, ":") if len(splitted) != 2 { log.Fatal("Wrong auth format") } passwords[splitted[0]] = splitted[1] } log.Println("root:", *root, "bind:", *bind) http.HandleFunc(*simpleURLPath, handler) http.HandleFunc(*refreshURLPath, handler) log.Fatal(http.ListenAndServe(*bind, nil)) }