main.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
   4               2019-2021 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "crypto/tls"
  27         "encoding/hex"
  28         "errors"
  29         "flag"
  30         "fmt"
  31         "io/ioutil"
  32         "log"
  33         "net"
  34         "net/http"
  35         "net/url"
  36         "os"
  37         "os/signal"
  38         "path/filepath"
  39         "regexp"
  40         "runtime"
  41         "strings"
  42         "syscall"
  43         "time"
  44
  45         "golang.org/x/net/netutil"
  46 )
  47
  48 const (
  49         Version   = "3.0.0"
  50         HTMLBegin = `<!DOCTYPE html>
  51 <html>
  52   <head>
  53     <title>Links for %s</title>
  54   </head>
  55   <body>
  56 `
  57         HTMLEnd      = "  </body>\n</html>\n"
  58         HTMLElement  = "    <a href=\"%s\"%s>%s</a><br/>\n"
  59         InternalFlag = ".internal"
  60         GPGSigExt    = ".asc"
  61
  62         Warranty = `This program is free software: you can redistribute it and/or modify
  63 it under the terms of the GNU General Public License as published by
  64 the Free Software Foundation, version 3 of the License.
  65
  66 This program is distributed in the hope that it will be useful,
  67 but WITHOUT ANY WARRANTY; without even the implied warranty of
  68 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  69 GNU General Public License for more details.
  70
  71 You should have received a copy of the GNU General Public License
  72 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  73 )
  74
  75 const (
  76         HashAlgoSHA256     = "sha256"
  77         HashAlgoBLAKE2b256 = "blake2_256"
  78         HashAlgoSHA512     = "sha512"
  79         HashAlgoMD5        = "md5"
  80 )
  81
  82 var (
  83         pkgPyPI         = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  84         normalizationRe = regexp.MustCompilePOSIX("[-_.]+")
  85
  86         knownHashAlgos []string = []string{
  87                 HashAlgoSHA256,
  88                 HashAlgoBLAKE2b256,
  89                 HashAlgoSHA512,
  90                 HashAlgoMD5,
  91         }
  92
  93         root       = flag.String("root", "./packages", "Path to packages directory")
  94         bind       = flag.String("bind", "[::]:8080", "Address to bind to")
  95         maxClients = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  96         doUCSPI    = flag.Bool("ucspi", false, "Work as UCSPI-TCP service")
  97
  98         tlsCert = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
  99         tlsKey  = flag.String("tls-key", "", "Path to TLS X.509 private key")
 100
 101         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
 102         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
 103         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
 104
 105         pypiURL      = flag.String("pypi", "https://pypi.org/simple/", "Upstream (PyPI) URL")
 106         pypiCertHash = flag.String("pypi-cert-hash", "", "Authenticate upstream by its X.509 certificate's SPKI SHA256 hash")
 107
 108         passwdPath  = flag.String("passwd", "", "Path to FIFO for upload authentication")
 109         passwdCheck = flag.Bool("passwd-check", false, "Run password checker")
 110
 111         logTimestamped = flag.Bool("log-timestamped", false, "Prepend timestmap to log messages")
 112         fsck           = flag.Bool("fsck", false, "Check integrity of all packages (errors are in stderr)")
 113         version        = flag.Bool("version", false, "Print version information")
 114         warranty       = flag.Bool("warranty", false, "Print warranty information")
 115
 116         killed        bool
 117         pypiURLParsed *url.URL
 118 )
 119
 120 func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool {
 121         path := filepath.Join(*root, pkgName)
 122         if _, err := os.Stat(path); os.IsNotExist(err) {
 123                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
 124                         log.Println("error", r.RemoteAddr, "mkdir", pkgName, err)
 125                         http.Error(w, err.Error(), http.StatusInternalServerError)
 126                         return false
 127                 }
 128                 log.Println(r.RemoteAddr, "mkdir", pkgName)
 129         }
 130         return true
 131 }
 132
 133 func listRoot(w http.ResponseWriter, r *http.Request) {
 134         files, err := ioutil.ReadDir(*root)
 135         if err != nil {
 136                 log.Println("error", r.RemoteAddr, "root", err)
 137                 http.Error(w, err.Error(), http.StatusInternalServerError)
 138                 return
 139         }
 140         var result bytes.Buffer
 141         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
 142         for _, file := range files {
 143                 if file.Mode().IsDir() {
 144                         result.WriteString(fmt.Sprintf(
 145                                 HTMLElement,
 146                                 *refreshURLPath+file.Name()+"/",
 147                                 "", file.Name(),
 148                         ))
 149                 }
 150         }
 151         result.WriteString(HTMLEnd)
 152         w.Write(result.Bytes())
 153 }
 154
 155 func listDir(
 156         w http.ResponseWriter,
 157         r *http.Request,
 158         pkgName string,
 159         autorefresh, gpgUpdate bool,
 160 ) {
 161         dirPath := filepath.Join(*root, pkgName)
 162         if autorefresh {
 163                 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
 164                         return
 165                 }
 166         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "", false) {
 167                 return
 168         }
 169         fis, err := ioutil.ReadDir(dirPath)
 170         if err != nil {
 171                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 172                 http.Error(w, err.Error(), http.StatusInternalServerError)
 173                 return
 174         }
 175         files := make(map[string]struct{}, len(fis)/2)
 176         for _, fi := range fis {
 177                 files[fi.Name()] = struct{}{}
 178         }
 179         var result bytes.Buffer
 180         result.WriteString(fmt.Sprintf(HTMLBegin, pkgName))
 181         for _, algo := range knownHashAlgos {
 182                 for fn := range files {
 183                         if killed {
 184                                 // Skip expensive I/O when shutting down
 185                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 186                                 return
 187                         }
 188                         if !strings.HasSuffix(fn, "."+algo) {
 189                                 continue
 190                         }
 191                         delete(files, fn)
 192                         digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
 193                         if err != nil {
 194                                 log.Println("error", r.RemoteAddr, "list", fn, err)
 195                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 196                                 return
 197                         }
 198                         fnClean := strings.TrimSuffix(fn, "."+algo)
 199                         delete(files, fnClean)
 200                         gpgSigAttr := ""
 201                         if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil {
 202                                 gpgSigAttr = " data-gpg-sig=true"
 203                                 delete(files, fnClean+GPGSigExt)
 204                         }
 205                         result.WriteString(fmt.Sprintf(
 206                                 HTMLElement,
 207                                 strings.Join([]string{
 208                                         *refreshURLPath, pkgName, "/", fnClean,
 209                                         "#", algo, "=", hex.EncodeToString(digest),
 210                                 }, ""),
 211                                 gpgSigAttr,
 212                                 fnClean,
 213                         ))
 214                 }
 215         }
 216         result.WriteString(HTMLEnd)
 217         w.Write(result.Bytes())
 218 }
 219
 220 func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) {
 221         log.Println(r.RemoteAddr, "get", filename)
 222         path := filepath.Join(*root, pkgName, filename)
 223         if _, err := os.Stat(path); os.IsNotExist(err) {
 224                 if !refreshDir(w, r, pkgName, filename, false) {
 225                         return
 226                 }
 227         }
 228         http.ServeFile(w, r, path)
 229 }
 230
 231 func handler(w http.ResponseWriter, r *http.Request) {
 232         switch r.Method {
 233         case "GET":
 234                 var path string
 235                 var autorefresh bool
 236                 var gpgUpdate bool
 237                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 238                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 239                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
 240                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 241                         autorefresh = true
 242                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
 243                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
 244                         autorefresh = true
 245                         gpgUpdate = true
 246                 } else {
 247                         http.Error(w, "unknown action", http.StatusBadRequest)
 248                         return
 249                 }
 250                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 251                 if len(parts) > 2 {
 252                         http.Error(w, "invalid path", http.StatusBadRequest)
 253                         return
 254                 }
 255                 if len(parts) == 1 {
 256                         if parts[0] == "" {
 257                                 listRoot(w, r)
 258                         } else {
 259                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
 260                         }
 261                 } else {
 262                         servePkg(w, r, parts[0], parts[1])
 263                 }
 264         case "POST":
 265                 serveUpload(w, r)
 266         default:
 267                 http.Error(w, "unknown action", http.StatusBadRequest)
 268         }
 269 }
 270
 271 func main() {
 272         flag.Parse()
 273         if *warranty {
 274                 fmt.Println(Warranty)
 275                 return
 276         }
 277         if *version {
 278                 fmt.Println("GoCheese", Version, "built with", runtime.Version())
 279                 return
 280         }
 281
 282         if *logTimestamped {
 283                 log.SetFlags(log.Ldate | log.Lmicroseconds | log.Lshortfile)
 284         } else {
 285                 log.SetFlags(log.Lshortfile)
 286         }
 287         if !*doUCSPI {
 288                 log.SetOutput(os.Stdout)
 289         }
 290
 291         if *fsck {
 292                 if !goodIntegrity() {
 293                         os.Exit(1)
 294                 }
 295                 return
 296         }
 297
 298         if *passwdCheck {
 299                 if passwdReader(os.Stdin) {
 300                         os.Exit(0)
 301                 } else {
 302                         os.Exit(1)
 303                 }
 304         }
 305
 306         if *passwdPath != "" {
 307                 go func() {
 308                         for {
 309                                 fd, err := os.OpenFile(*passwdPath, os.O_RDONLY, os.FileMode(0666))
 310                                 if err != nil {
 311                                         log.Fatalln(err)
 312                                 }
 313                                 passwdReader(fd)
 314                                 fd.Close()
 315                         }
 316                 }()
 317         }
 318
 319         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 320                 log.Fatalln("Both -tls-cert and -tls-key are required")
 321         }
 322
 323         var err error
 324         pypiURLParsed, err = url.Parse(*pypiURL)
 325         if err != nil {
 326                 log.Fatalln(err)
 327         }
 328         tlsConfig := tls.Config{
 329                 ClientSessionCache: tls.NewLRUClientSessionCache(16),
 330                 NextProtos:         []string{"h2", "http/1.1"},
 331         }
 332         pypiHTTPTransport = http.Transport{
 333                 ForceAttemptHTTP2: true,
 334                 TLSClientConfig:   &tlsConfig,
 335         }
 336         if *pypiCertHash != "" {
 337                 ourDgst, err := hex.DecodeString(*pypiCertHash)
 338                 if err != nil {
 339                         log.Fatalln(err)
 340                 }
 341                 tlsConfig.VerifyConnection = func(s tls.ConnectionState) error {
 342                         spki := s.VerifiedChains[0][0].RawSubjectPublicKeyInfo
 343                         theirDgst := sha256.Sum256(spki)
 344                         if bytes.Compare(ourDgst, theirDgst[:]) != 0 {
 345                                 return errors.New("certificate's SPKI digest mismatch")
 346                         }
 347                         return nil
 348                 }
 349         }
 350
 351         server := &http.Server{
 352                 ReadTimeout:  time.Minute,
 353                 WriteTimeout: time.Minute,
 354         }
 355         http.HandleFunc(*norefreshURLPath, handler)
 356         http.HandleFunc(*refreshURLPath, handler)
 357         if *gpgUpdateURLPath != "" {
 358                 http.HandleFunc(*gpgUpdateURLPath, handler)
 359         }
 360
 361         if *doUCSPI {
 362                 server.SetKeepAlivesEnabled(false)
 363                 ln := &UCSPI{}
 364                 server.ConnState = connStater
 365                 err := server.Serve(ln)
 366                 if _, ok := err.(UCSPIAlreadyAccepted); !ok {
 367                         log.Fatalln(err)
 368                 }
 369                 UCSPIJob.Wait()
 370                 return
 371         }
 372
 373         ln, err := net.Listen("tcp", *bind)
 374         if err != nil {
 375                 log.Fatal(err)
 376         }
 377         ln = netutil.LimitListener(ln, *maxClients)
 378
 379         needsShutdown := make(chan os.Signal, 0)
 380         exitErr := make(chan error, 0)
 381         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 382         go func(s *http.Server) {
 383                 <-needsShutdown
 384                 killed = true
 385                 log.Println("shutting down")
 386                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 387                 exitErr <- s.Shutdown(ctx)
 388                 cancel()
 389         }(server)
 390
 391         log.Println(
 392                 "GoCheese", Version, "listens:",
 393                 "root:", *root,
 394                 "bind:", *bind,
 395                 "pypi:", *pypiURL,
 396         )
 397         if *tlsCert == "" {
 398                 err = server.Serve(ln)
 399         } else {
 400                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 401         }
 402         if err != http.ErrServerClosed {
 403                 log.Fatal(err)
 404         }
 405         if err := <-exitErr; err != nil {
 406                 log.Fatal(err)
 407         }
 408 }