gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2020 Sergey Matveev <stargrave@stargrave.org>
   4               2019-2020 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "encoding/hex"
  26         "flag"
  27         "fmt"
  28         "io/ioutil"
  29         "log"
  30         "net"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "os/signal"
  35         "path/filepath"
  36         "regexp"
  37         "runtime"
  38         "strings"
  39         "syscall"
  40         "time"
  41
  42         "golang.org/x/net/netutil"
  43 )
  44
  45 const (
  46         Version   = "2.5.0"
  47         HTMLBegin = `<!DOCTYPE html>
  48 <html>
  49   <head>
  50     <title>Links for %s</title>
  51   </head>
  52   <body>
  53 `
  54         HTMLEnd      = "  </body>\n</html>\n"
  55         HTMLElement  = "    <a href=\"%s\"%s>%s</a><br/>\n"
  56         InternalFlag = ".internal"
  57         GPGSigExt    = ".asc"
  58
  59         Warranty = `This program is free software: you can redistribute it and/or modify
  60 it under the terms of the GNU General Public License as published by
  61 the Free Software Foundation, version 3 of the License.
  62
  63 This program is distributed in the hope that it will be useful,
  64 but WITHOUT ANY WARRANTY; without even the implied warranty of
  65 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  66 GNU General Public License for more details.
  67
  68 You should have received a copy of the GNU General Public License
  69 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  70 )
  71
  72 const (
  73         HashAlgoSHA256     = "sha256"
  74         HashAlgoBLAKE2b256 = "blake2_256"
  75         HashAlgoSHA512     = "sha512"
  76         HashAlgoMD5        = "md5"
  77 )
  78
  79 var (
  80         pkgPyPI         = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  81         normalizationRe = regexp.MustCompilePOSIX("[-_.]+")
  82
  83         knownHashAlgos []string = []string{
  84                 HashAlgoSHA256,
  85                 HashAlgoBLAKE2b256,
  86                 HashAlgoSHA512,
  87                 HashAlgoMD5,
  88         }
  89
  90         root             = flag.String("root", "./packages", "Path to packages directory")
  91         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  92         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
  93         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
  94         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  95         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  96         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
  97         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  98         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  99         logTimestamped   = flag.Bool("log-timestamped", false, "Prepend timestmap to log messages")
 100         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
 101         fsck             = flag.Bool("fsck", false, "Check integrity of all packages (errors are in stderr)")
 102         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
 103         version          = flag.Bool("version", false, "Print version information")
 104         warranty         = flag.Bool("warranty", false, "Print warranty information")
 105
 106         killed        bool
 107         pypiURLParsed *url.URL
 108 )
 109
 110 func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool {
 111         path := filepath.Join(*root, pkgName)
 112         if _, err := os.Stat(path); os.IsNotExist(err) {
 113                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
 114                         log.Println("error", r.RemoteAddr, "mkdir", pkgName, err)
 115                         http.Error(w, err.Error(), http.StatusInternalServerError)
 116                         return false
 117                 }
 118                 log.Println(r.RemoteAddr, "mkdir", pkgName)
 119         }
 120         return true
 121 }
 122
 123 func listRoot(w http.ResponseWriter, r *http.Request) {
 124         files, err := ioutil.ReadDir(*root)
 125         if err != nil {
 126                 log.Println("error", r.RemoteAddr, "root", err)
 127                 http.Error(w, err.Error(), http.StatusInternalServerError)
 128                 return
 129         }
 130         var result bytes.Buffer
 131         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
 132         for _, file := range files {
 133                 if file.Mode().IsDir() {
 134                         result.WriteString(fmt.Sprintf(
 135                                 HTMLElement,
 136                                 *refreshURLPath+file.Name()+"/",
 137                                 "", file.Name(),
 138                         ))
 139                 }
 140         }
 141         result.WriteString(HTMLEnd)
 142         w.Write(result.Bytes())
 143 }
 144
 145 func listDir(
 146         w http.ResponseWriter,
 147         r *http.Request,
 148         pkgName string,
 149         autorefresh, gpgUpdate bool,
 150 ) {
 151         dirPath := filepath.Join(*root, pkgName)
 152         if autorefresh {
 153                 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
 154                         return
 155                 }
 156         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "", false) {
 157                 return
 158         }
 159         fis, err := ioutil.ReadDir(dirPath)
 160         if err != nil {
 161                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 162                 http.Error(w, err.Error(), http.StatusInternalServerError)
 163                 return
 164         }
 165         files := make(map[string]struct{}, len(fis)/2)
 166         for _, fi := range fis {
 167                 files[fi.Name()] = struct{}{}
 168         }
 169         var result bytes.Buffer
 170         result.WriteString(fmt.Sprintf(HTMLBegin, pkgName))
 171         for _, algo := range knownHashAlgos {
 172                 for fn, _ := range files {
 173                         if killed {
 174                                 // Skip expensive I/O when shutting down
 175                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 176                                 return
 177                         }
 178                         if !strings.HasSuffix(fn, "."+algo) {
 179                                 continue
 180                         }
 181                         delete(files, fn)
 182                         digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
 183                         if err != nil {
 184                                 log.Println("error", r.RemoteAddr, "list", fn, err)
 185                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 186                                 return
 187                         }
 188                         fnClean := strings.TrimSuffix(fn, "."+algo)
 189                         delete(files, fnClean)
 190                         gpgSigAttr := ""
 191                         if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil {
 192                                 gpgSigAttr = " data-gpg-sig=true"
 193                                 delete(files, fnClean+GPGSigExt)
 194                         }
 195                         result.WriteString(fmt.Sprintf(
 196                                 HTMLElement,
 197                                 strings.Join([]string{
 198                                         *refreshURLPath, pkgName, "/", fnClean,
 199                                         "#", algo, "=", hex.EncodeToString(digest),
 200                                 }, ""),
 201                                 gpgSigAttr,
 202                                 fnClean,
 203                         ))
 204                 }
 205         }
 206         result.WriteString(HTMLEnd)
 207         w.Write(result.Bytes())
 208 }
 209
 210 func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) {
 211         log.Println(r.RemoteAddr, "get", filename)
 212         path := filepath.Join(*root, pkgName, filename)
 213         if _, err := os.Stat(path); os.IsNotExist(err) {
 214                 if !refreshDir(w, r, pkgName, filename, false) {
 215                         return
 216                 }
 217         }
 218         http.ServeFile(w, r, path)
 219 }
 220
 221 func handler(w http.ResponseWriter, r *http.Request) {
 222         switch r.Method {
 223         case "GET":
 224                 var path string
 225                 var autorefresh bool
 226                 var gpgUpdate bool
 227                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 228                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 229                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
 230                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 231                         autorefresh = true
 232                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
 233                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
 234                         autorefresh = true
 235                         gpgUpdate = true
 236                 } else {
 237                         http.Error(w, "unknown action", http.StatusBadRequest)
 238                         return
 239                 }
 240                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 241                 if len(parts) > 2 {
 242                         http.Error(w, "invalid path", http.StatusBadRequest)
 243                         return
 244                 }
 245                 if len(parts) == 1 {
 246                         if parts[0] == "" {
 247                                 listRoot(w, r)
 248                         } else {
 249                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
 250                         }
 251                 } else {
 252                         servePkg(w, r, parts[0], parts[1])
 253                 }
 254         case "POST":
 255                 serveUpload(w, r)
 256         default:
 257                 http.Error(w, "unknown action", http.StatusBadRequest)
 258         }
 259 }
 260
 261 func main() {
 262         flag.Parse()
 263         if *warranty {
 264                 fmt.Println(Warranty)
 265                 return
 266         }
 267         if *version {
 268                 fmt.Println("GoCheese", Version, "built with", runtime.Version())
 269                 return
 270         }
 271
 272         if *logTimestamped {
 273                 log.SetFlags(log.Ldate | log.Lmicroseconds | log.Lshortfile)
 274         } else {
 275                 log.SetFlags(log.Lshortfile)
 276         }
 277         log.SetOutput(os.Stdout)
 278
 279         if *fsck {
 280                 if !goodIntegrity() {
 281                         os.Exit(1)
 282                 }
 283                 return
 284         }
 285
 286         if *passwdCheck {
 287                 refreshPasswd()
 288                 return
 289         }
 290
 291         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 292                 log.Fatalln("Both -tls-cert and -tls-key are required")
 293         }
 294
 295         var err error
 296         pypiURLParsed, err = url.Parse(*pypiURL)
 297         if err != nil {
 298                 log.Fatalln(err)
 299         }
 300         refreshPasswd()
 301
 302         ln, err := net.Listen("tcp", *bind)
 303         if err != nil {
 304                 log.Fatal(err)
 305         }
 306         ln = netutil.LimitListener(ln, *maxClients)
 307         server := &http.Server{
 308                 ReadTimeout:  time.Minute,
 309                 WriteTimeout: time.Minute,
 310         }
 311         http.HandleFunc(*norefreshURLPath, handler)
 312         http.HandleFunc(*refreshURLPath, handler)
 313         if *gpgUpdateURLPath != "" {
 314                 http.HandleFunc(*gpgUpdateURLPath, handler)
 315         }
 316
 317         needsRefreshPasswd := make(chan os.Signal, 0)
 318         needsShutdown := make(chan os.Signal, 0)
 319         exitErr := make(chan error, 0)
 320         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 321         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 322         go func() {
 323                 for range needsRefreshPasswd {
 324                         log.Println("refreshing passwords")
 325                         refreshPasswd()
 326                 }
 327         }()
 328         go func(s *http.Server) {
 329                 <-needsShutdown
 330                 killed = true
 331                 log.Println("shutting down")
 332                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 333                 exitErr <- s.Shutdown(ctx)
 334                 cancel()
 335         }(server)
 336
 337         log.Println(
 338                 "GoCheese", Version, "listens:",
 339                 "root:", *root,
 340                 "bind:", *bind,
 341                 "pypi:", *pypiURL,
 342         )
 343         if *tlsCert == "" {
 344                 err = server.Serve(ln)
 345         } else {
 346                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 347         }
 348         if err != http.ErrServerClosed {
 349                 log.Fatal(err)
 350         }
 351         if err := <-exitErr; err != nil {
 352                 log.Fatal(err)
 353         }
 354 }