gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "encoding/hex"
  26         "flag"
  27         "fmt"
  28         "io/ioutil"
  29         "log"
  30         "net"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "os/signal"
  35         "path/filepath"
  36         "regexp"
  37         "runtime"
  38         "strings"
  39         "syscall"
  40         "time"
  41
  42         "golang.org/x/net/netutil"
  43 )
  44
  45 const (
  46         HTMLBegin = `<!DOCTYPE html>
  47 <html>
  48   <head>
  49     <title>Links for %s</title>
  50   </head>
  51   <body>
  52 `
  53         HTMLEnd      = "  </body>\n</html>\n"
  54         HTMLElement  = "    <a href=\"%s\"%s>%s</a><br/>\n"
  55         InternalFlag = ".internal"
  56         GPGSigExt    = ".asc"
  57
  58         Warranty = `This program is free software: you can redistribute it and/or modify
  59 it under the terms of the GNU General Public License as published by
  60 the Free Software Foundation, version 3 of the License.
  61
  62 This program is distributed in the hope that it will be useful,
  63 but WITHOUT ANY WARRANTY; without even the implied warranty of
  64 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  65 GNU General Public License for more details.
  66
  67 You should have received a copy of the GNU General Public License
  68 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  69 )
  70
  71 var (
  72         pkgPyPI         = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  73         normalizationRe = regexp.MustCompilePOSIX("[-_.]+")
  74
  75         HashAlgoSHA256              = "sha256"
  76         HashAlgoBLAKE2b256          = "blake2_256"
  77         HashAlgoSHA512              = "sha512"
  78         HashAlgoMD5                 = "md5"
  79         knownHashAlgos     []string = []string{
  80                 HashAlgoSHA256,
  81                 HashAlgoBLAKE2b256,
  82                 HashAlgoSHA512,
  83                 HashAlgoMD5,
  84         }
  85
  86         root             = flag.String("root", "./packages", "Path to packages directory")
  87         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  88         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
  89         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
  90         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  91         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  92         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
  93         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  94         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  95         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  96         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  97         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  98         version          = flag.Bool("version", false, "Print version information")
  99         warranty         = flag.Bool("warranty", false, "Print warranty information")
 100
 101         Version       string = "UNKNOWN"
 102         killed        bool
 103         pypiURLParsed *url.URL
 104 )
 105
 106 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
 107         path := filepath.Join(*root, dir)
 108         if _, err := os.Stat(path); os.IsNotExist(err) {
 109                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
 110                         http.Error(w, err.Error(), http.StatusInternalServerError)
 111                         return false
 112                 }
 113                 log.Println(r.RemoteAddr, "mkdir", dir)
 114         }
 115         return true
 116 }
 117
 118 func listRoot(w http.ResponseWriter, r *http.Request) {
 119         files, err := ioutil.ReadDir(*root)
 120         if err != nil {
 121                 http.Error(w, err.Error(), http.StatusInternalServerError)
 122                 return
 123         }
 124         var result bytes.Buffer
 125         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
 126         for _, file := range files {
 127                 if file.Mode().IsDir() {
 128                         result.WriteString(fmt.Sprintf(
 129                                 HTMLElement,
 130                                 *refreshURLPath+file.Name()+"/",
 131                                 file.Name(),
 132                         ))
 133                 }
 134         }
 135         result.WriteString(HTMLEnd)
 136         w.Write(result.Bytes())
 137 }
 138
 139 func listDir(
 140         w http.ResponseWriter,
 141         r *http.Request,
 142         dir string,
 143         autorefresh,
 144         gpgUpdate bool,
 145 ) {
 146         dirPath := filepath.Join(*root, dir)
 147         if autorefresh {
 148                 if !refreshDir(w, r, dir, "", gpgUpdate) {
 149                         return
 150                 }
 151         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) {
 152                 return
 153         }
 154         fis, err := ioutil.ReadDir(dirPath)
 155         if err != nil {
 156                 http.Error(w, err.Error(), http.StatusInternalServerError)
 157                 return
 158         }
 159         files := make(map[string]struct{}, len(fis)/2)
 160         for _, fi := range fis {
 161                 files[fi.Name()] = struct{}{}
 162         }
 163         var result bytes.Buffer
 164         result.WriteString(fmt.Sprintf(HTMLBegin, dir))
 165         for _, algo := range knownHashAlgos {
 166                 for fn, _ := range files {
 167                         if killed {
 168                                 // Skip expensive I/O when shutting down
 169                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 170                                 return
 171                         }
 172                         if !strings.HasSuffix(fn, "."+algo) {
 173                                 continue
 174                         }
 175                         delete(files, fn)
 176                         digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
 177                         if err != nil {
 178                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 179                                 return
 180                         }
 181                         fnClean := strings.TrimSuffix(fn, "."+algo)
 182                         delete(files, fnClean)
 183                         gpgSigAttr := ""
 184                         if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil {
 185                                 gpgSigAttr = " data-gpg-sig=true"
 186                                 delete(files, fnClean+GPGSigExt)
 187                         }
 188                         result.WriteString(fmt.Sprintf(
 189                                 HTMLElement,
 190                                 strings.Join([]string{
 191                                         *refreshURLPath, dir, "/", fnClean,
 192                                         "#", algo, "=", hex.EncodeToString(digest),
 193                                 }, ""),
 194                                 gpgSigAttr,
 195                                 fnClean,
 196                         ))
 197                 }
 198         }
 199         result.WriteString(HTMLEnd)
 200         w.Write(result.Bytes())
 201 }
 202
 203 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 204         log.Println(r.RemoteAddr, "get", filename)
 205         path := filepath.Join(*root, dir, filename)
 206         if _, err := os.Stat(path); os.IsNotExist(err) {
 207                 if !refreshDir(w, r, dir, filename, false) {
 208                         return
 209                 }
 210         }
 211         http.ServeFile(w, r, path)
 212 }
 213
 214 func handler(w http.ResponseWriter, r *http.Request) {
 215         switch r.Method {
 216         case "GET":
 217                 var path string
 218                 var autorefresh bool
 219                 var gpgUpdate bool
 220                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 221                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 222                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
 223                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 224                         autorefresh = true
 225                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
 226                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
 227                         autorefresh = true
 228                         gpgUpdate = true
 229                 } else {
 230                         http.Error(w, "unknown action", http.StatusBadRequest)
 231                         return
 232                 }
 233                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 234                 if len(parts) > 2 {
 235                         http.Error(w, "invalid path", http.StatusBadRequest)
 236                         return
 237                 }
 238                 if len(parts) == 1 {
 239                         if parts[0] == "" {
 240                                 listRoot(w, r)
 241                         } else {
 242                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
 243                         }
 244                 } else {
 245                         servePkg(w, r, parts[0], parts[1])
 246                 }
 247         case "POST":
 248                 serveUpload(w, r)
 249         default:
 250                 http.Error(w, "unknown action", http.StatusBadRequest)
 251         }
 252 }
 253
 254 func main() {
 255         flag.Parse()
 256         if *warranty {
 257                 fmt.Println(Warranty)
 258                 return
 259         }
 260         if *version {
 261                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 262                 return
 263         }
 264         if *fsck {
 265                 if !goodIntegrity() {
 266                         os.Exit(1)
 267                 }
 268                 return
 269         }
 270         if *passwdCheck {
 271                 refreshPasswd()
 272                 return
 273         }
 274         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 275                 log.Fatalln("Both -tls-cert and -tls-key are required")
 276         }
 277         var err error
 278         pypiURLParsed, err = url.Parse(*pypiURL)
 279         if err != nil {
 280                 log.Fatalln(err)
 281         }
 282         refreshPasswd()
 283         log.Println("root:", *root, "bind:", *bind)
 284
 285         ln, err := net.Listen("tcp", *bind)
 286         if err != nil {
 287                 log.Fatal(err)
 288         }
 289         ln = netutil.LimitListener(ln, *maxClients)
 290         server := &http.Server{
 291                 ReadTimeout:  time.Minute,
 292                 WriteTimeout: time.Minute,
 293         }
 294         http.HandleFunc(*norefreshURLPath, handler)
 295         http.HandleFunc(*refreshURLPath, handler)
 296         if *gpgUpdateURLPath != "" {
 297                 http.HandleFunc(*gpgUpdateURLPath, handler)
 298         }
 299
 300         needsRefreshPasswd := make(chan os.Signal, 0)
 301         needsShutdown := make(chan os.Signal, 0)
 302         exitErr := make(chan error, 0)
 303         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 304         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 305         go func() {
 306                 for range needsRefreshPasswd {
 307                         log.Println("Refreshing passwords")
 308                         refreshPasswd()
 309                 }
 310         }()
 311         go func(s *http.Server) {
 312                 <-needsShutdown
 313                 killed = true
 314                 log.Println("Shutting down")
 315                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 316                 exitErr <- s.Shutdown(ctx)
 317                 cancel()
 318         }(server)
 319
 320         if *tlsCert == "" {
 321                 err = server.Serve(ln)
 322         } else {
 323                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 324         }
 325         if err != http.ErrServerClosed {
 326                 log.Fatal(err)
 327         }
 328         if err := <-exitErr; err != nil {
 329                 log.Fatal(err)
 330         }
 331 }