Links for %s

X-Git-Url: http://www.git.cypherpunks.ru/?p=gocheese.git;a=blobdiff_plain;f=main.go;h=08c75535c1491feb8ac230f754cb37b63086e4c2;hp=12a325a93435af81d330f2ff9e844e31da891e13;hb=HEAD;hpb=168de6871afdd60f2bc1c529170033ef56307d7c diff --git a/main.go b/main.go index 12a325a..cdab06e 100644 --- a/main.go +++ b/main.go @@ -1,20 +1,18 @@ -/* -GoCheese -- Python private package repository and caching proxy -Copyright (C) 2019-2021 Sergey Matveev - 2019-2021 Elena Balakhonova - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ +// GoCheese -- Python private package repository and caching proxy +// Copyright (C) 2019-2024 Sergey Matveev +// 2019-2024 Elena Balakhonova +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . // Python private package repository and caching proxy package main @@ -28,7 +26,6 @@ import ( "errors" "flag" "fmt" - "io/ioutil" "log" "net" "net/http" @@ -36,7 +33,6 @@ import ( "os" "os/signal" "path/filepath" - "regexp" "runtime" "strings" "syscall" @@ -46,182 +42,45 @@ import ( ) const ( - Version = "3.0.0" - HTMLBegin = ` - - - Links for %s - - -` - HTMLEnd = " \n\n" - HTMLElement = " %s
\n" - InternalFlag = ".internal" - GPGSigExt = ".asc" - - Warranty = `This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see .` -) - -const ( - HashAlgoSHA256 = "sha256" - HashAlgoBLAKE2b256 = "blake2_256" - HashAlgoSHA512 = "sha512" - HashAlgoMD5 = "md5" + Version = "4.2.0" + UserAgent = "GoCheese/" + Version ) var ( - pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) - normalizationRe = regexp.MustCompilePOSIX("[-_.]+") - - knownHashAlgos []string = []string{ - HashAlgoSHA256, - HashAlgoBLAKE2b256, - HashAlgoSHA512, - HashAlgoMD5, - } + Root string + Bind = flag.String("bind", DefaultBind, "") + MaxClients = flag.Int("maxclients", DefaultMaxClients, "") + DoUCSPI = flag.Bool("ucspi", false, "") - root = flag.String("root", "./packages", "Path to packages directory") - bind = flag.String("bind", "[::]:8080", "Address to bind to") - maxClients = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients") - doUCSPI = flag.Bool("ucspi", false, "Work as UCSPI-TCP service") + TLSCert = flag.String("tls-cert", "", "") + TLSKey = flag.String("tls-key", "", "") - tlsCert = flag.String("tls-cert", "", "Path to TLS X.509 certificate") - tlsKey = flag.String("tls-key", "", "Path to TLS X.509 private key") + NoRefreshURLPath = flag.String("norefresh", DefaultNoRefreshURLPath, "") + RefreshURLPath = flag.String("refresh", DefaultRefreshURLPath, "") + JSONURLPath = flag.String("json", DefaultJSONURLPath, "") - norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path") - refreshURLPath = flag.String("refresh", "/simple/", "Auto-refreshing URL path") - gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path") + PyPIURL = flag.String("pypi", DefaultPyPIURL, "") + JSONURL = flag.String("pypi-json", DefaultJSONURL, "") + PyPICertHash = flag.String("pypi-cert-hash", "", "") - pypiURL = flag.String("pypi", "https://pypi.org/simple/", "Upstream (PyPI) URL") - pypiCertHash = flag.String("pypi-cert-hash", "", "Authenticate upstream by its X.509 certificate's SPKI SHA256 hash") + PasswdPath = flag.String("passwd", "", "") + PasswdListPath = flag.String("passwd-list", "", "") + PasswdCheck = flag.Bool("passwd-check", false, "") + AuthRequired = flag.Bool("auth-required", false, "") - passwdPath = flag.String("passwd", "", "Path to FIFO for upload authentication") - passwdCheck = flag.Bool("passwd-check", false, "Run password checker") + LogTimestamped = flag.Bool("log-timestamped", false, "") + FSCK = flag.Bool("fsck", false, "") + DoVersion = flag.Bool("version", false, "") + DoWarranty = flag.Bool("warranty", false, "") - logTimestamped = flag.Bool("log-timestamped", false, "Prepend timestmap to log messages") - fsck = flag.Bool("fsck", false, "Check integrity of all packages (errors are in stderr)") - version = flag.Bool("version", false, "Print version information") - warranty = flag.Bool("warranty", false, "Print warranty information") - - killed bool - pypiURLParsed *url.URL + Killed bool ) -func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool { - path := filepath.Join(*root, pkgName) - if _, err := os.Stat(path); os.IsNotExist(err) { - if err = os.Mkdir(path, os.FileMode(0777)); err != nil { - log.Println("error", r.RemoteAddr, "mkdir", pkgName, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "mkdir", pkgName) - } - return true -} - -func listRoot(w http.ResponseWriter, r *http.Request) { - files, err := ioutil.ReadDir(*root) - if err != nil { - log.Println("error", r.RemoteAddr, "root", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - var result bytes.Buffer - result.WriteString(fmt.Sprintf(HTMLBegin, "root")) - for _, file := range files { - if file.Mode().IsDir() { - result.WriteString(fmt.Sprintf( - HTMLElement, - *refreshURLPath+file.Name()+"/", - "", file.Name(), - )) - } - } - result.WriteString(HTMLEnd) - w.Write(result.Bytes()) -} - -func listDir( - w http.ResponseWriter, - r *http.Request, - pkgName string, - autorefresh, gpgUpdate bool, -) { - dirPath := filepath.Join(*root, pkgName) - if autorefresh { - if !refreshDir(w, r, pkgName, "", gpgUpdate) { - return - } - } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "", false) { - return - } - fis, err := ioutil.ReadDir(dirPath) - if err != nil { - log.Println("error", r.RemoteAddr, "list", pkgName, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - files := make(map[string]struct{}, len(fis)/2) - for _, fi := range fis { - files[fi.Name()] = struct{}{} - } - var result bytes.Buffer - result.WriteString(fmt.Sprintf(HTMLBegin, pkgName)) - for _, algo := range knownHashAlgos { - for fn := range files { - if killed { - // Skip expensive I/O when shutting down - http.Error(w, "shutting down", http.StatusInternalServerError) - return - } - if !strings.HasSuffix(fn, "."+algo) { - continue - } - delete(files, fn) - digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn)) - if err != nil { - log.Println("error", r.RemoteAddr, "list", fn, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - fnClean := strings.TrimSuffix(fn, "."+algo) - delete(files, fnClean) - gpgSigAttr := "" - if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil { - gpgSigAttr = " data-gpg-sig=true" - delete(files, fnClean+GPGSigExt) - } - result.WriteString(fmt.Sprintf( - HTMLElement, - strings.Join([]string{ - *refreshURLPath, pkgName, "/", fnClean, - "#", algo, "=", hex.EncodeToString(digest), - }, ""), - gpgSigAttr, - fnClean, - )) - } - } - result.WriteString(HTMLEnd) - w.Write(result.Bytes()) -} - func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) { log.Println(r.RemoteAddr, "get", filename) - path := filepath.Join(*root, pkgName, filename) + path := filepath.Join(Root, pkgName, filename) if _, err := os.Stat(path); os.IsNotExist(err) { - if !refreshDir(w, r, pkgName, filename, false) { + if !refreshDir(w, r, pkgName, filename) { return } } @@ -229,20 +88,16 @@ func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) } func handler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", UserAgent) switch r.Method { case "GET": var path string var autorefresh bool - var gpgUpdate bool - if strings.HasPrefix(r.URL.Path, *norefreshURLPath) { - path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath) - } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) { - path = strings.TrimPrefix(r.URL.Path, *refreshURLPath) - autorefresh = true - } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) { - path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath) + if strings.HasPrefix(r.URL.Path, *NoRefreshURLPath) { + path = strings.TrimPrefix(r.URL.Path, *NoRefreshURLPath) + } else if strings.HasPrefix(r.URL.Path, *RefreshURLPath) { + path = strings.TrimPrefix(r.URL.Path, *RefreshURLPath) autorefresh = true - gpgUpdate = true } else { http.Error(w, "unknown action", http.StatusBadRequest) return @@ -256,7 +111,7 @@ func handler(w http.ResponseWriter, r *http.Request) { if parts[0] == "" { listRoot(w, r) } else { - listDir(w, r, parts[0], autorefresh, gpgUpdate) + serveListDir(w, r, parts[0], autorefresh) } } else { servePkg(w, r, parts[0], parts[1]) @@ -269,33 +124,43 @@ func handler(w http.ResponseWriter, r *http.Request) { } func main() { + flag.Usage = usage flag.Parse() - if *warranty { + if *DoWarranty { fmt.Println(Warranty) return } - if *version { + if *DoVersion { fmt.Println("GoCheese", Version, "built with", runtime.Version()) return } - if *logTimestamped { + if *LogTimestamped { log.SetFlags(log.Ldate | log.Lmicroseconds | log.Lshortfile) } else { log.SetFlags(log.Lshortfile) } - if !*doUCSPI { + if !*DoUCSPI { log.SetOutput(os.Stdout) } - if *fsck { + if len(flag.Args()) != 1 { + usage() + os.Exit(1) + } + Root = flag.Args()[0] + if _, err := os.Stat(Root); err != nil { + log.Fatal(err) + } + + if *FSCK { if !goodIntegrity() { os.Exit(1) } return } - if *passwdCheck { + if *PasswdCheck { if passwdReader(os.Stdin) { os.Exit(0) } else { @@ -303,45 +168,68 @@ func main() { } } - if *passwdPath != "" { + if *PasswdPath != "" { go func() { for { - fd, err := os.OpenFile(*passwdPath, os.O_RDONLY, os.FileMode(0666)) + fd, err := os.OpenFile( + *PasswdPath, + os.O_RDONLY, + os.FileMode(0666), + ) if err != nil { - log.Fatalln(err) + log.Fatal(err) } passwdReader(fd) fd.Close() } }() } + if *PasswdListPath != "" { + go func() { + for { + fd, err := os.OpenFile( + *PasswdListPath, + os.O_WRONLY|os.O_APPEND, + os.FileMode(0666), + ) + if err != nil { + log.Fatal(err) + } + passwdLister(fd) + fd.Close() + } + }() + } - if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") { - log.Fatalln("Both -tls-cert and -tls-key are required") + if (*TLSCert != "" && *TLSKey == "") || (*TLSCert == "" && *TLSKey != "") { + log.Fatal("Both -tls-cert and -tls-key are required") } + UmaskCur = syscall.Umask(0) + syscall.Umask(UmaskCur) + var err error - pypiURLParsed, err = url.Parse(*pypiURL) + PyPIURLParsed, err = url.Parse(*PyPIURL) if err != nil { - log.Fatalln(err) + log.Fatal(err) } tlsConfig := tls.Config{ ClientSessionCache: tls.NewLRUClientSessionCache(16), NextProtos: []string{"h2", "http/1.1"}, } - pypiHTTPTransport = http.Transport{ + PyPIHTTPTransport = http.Transport{ ForceAttemptHTTP2: true, TLSClientConfig: &tlsConfig, } - if *pypiCertHash != "" { - ourDgst, err := hex.DecodeString(*pypiCertHash) + if *PyPICertHash != "" { + ourDgst, err := hex.DecodeString(*PyPICertHash) if err != nil { - log.Fatalln(err) + log.Fatal(err) } tlsConfig.VerifyConnection = func(s tls.ConnectionState) error { spki := s.VerifiedChains[0][0].RawSubjectPublicKeyInfo theirDgst := sha256.Sum256(spki) - if bytes.Compare(ourDgst, theirDgst[:]) != 0 { + if !bytes.Equal(ourDgst, theirDgst[:]) { return errors.New("certificate's SPKI digest mismatch") } return nil @@ -352,36 +240,36 @@ func main() { ReadTimeout: time.Minute, WriteTimeout: time.Minute, } - http.HandleFunc(*norefreshURLPath, handler) - http.HandleFunc(*refreshURLPath, handler) - if *gpgUpdateURLPath != "" { - http.HandleFunc(*gpgUpdateURLPath, handler) - } + http.HandleFunc("/", checkAuth(serveHRRoot)) + http.HandleFunc("/hr/", checkAuth(serveHRPkg)) + http.HandleFunc(*JSONURLPath, checkAuth(serveJSON)) + http.HandleFunc(*NoRefreshURLPath, checkAuth(handler)) + http.HandleFunc(*RefreshURLPath, checkAuth(handler)) - if *doUCSPI { + if *DoUCSPI { server.SetKeepAlivesEnabled(false) ln := &UCSPI{} server.ConnState = connStater err := server.Serve(ln) if _, ok := err.(UCSPIAlreadyAccepted); !ok { - log.Fatalln(err) + log.Fatal(err) } UCSPIJob.Wait() return } - ln, err := net.Listen("tcp", *bind) + ln, err := net.Listen("tcp", *Bind) if err != nil { log.Fatal(err) } - ln = netutil.LimitListener(ln, *maxClients) + ln = netutil.LimitListener(ln, *MaxClients) - needsShutdown := make(chan os.Signal, 0) - exitErr := make(chan error, 0) + needsShutdown := make(chan os.Signal, 1) + exitErr := make(chan error) signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT) go func(s *http.Server) { <-needsShutdown - killed = true + Killed = true log.Println("shutting down") ctx, cancel := context.WithTimeout(context.TODO(), time.Minute) exitErr <- s.Shutdown(ctx) @@ -389,15 +277,16 @@ func main() { }(server) log.Println( - "GoCheese", Version, "listens:", - "root:", *root, - "bind:", *bind, - "pypi:", *pypiURL, + UserAgent, "ready:", + "root:", Root, + "bind:", *Bind, + "pypi:", *PyPIURL, + "json:", *JSONURL, ) - if *tlsCert == "" { + if *TLSCert == "" { err = server.Serve(ln) } else { - err = server.ServeTLS(ln, *tlsCert, *tlsKey) + err = server.ServeTLS(ln, *TLSCert, *TLSKey) } if err != http.ErrServerClosed { log.Fatal(err)