main.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
   4               2019-2021 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "crypto/tls"
  27         "encoding/hex"
  28         "errors"
  29         "flag"
  30         "fmt"
  31         "io/ioutil"
  32         "log"
  33         "net"
  34         "net/http"
  35         "net/url"
  36         "os"
  37         "os/signal"
  38         "path/filepath"
  39         "regexp"
  40         "runtime"
  41         "strings"
  42         "syscall"
  43         "time"
  44
  45         "golang.org/x/net/netutil"
  46 )
  47
  48 const (
  49         Version   = "3.0.0"
  50         HTMLBegin = `<!DOCTYPE html>
  51 <html>
  52   <head>
  53     <meta name="pypi:repository-version" content="1.0">
  54     <title>Links for %s</title>
  55   </head>
  56   <body>
  57 `
  58         HTMLEnd      = "  </body>\n</html>\n"
  59         HTMLElement  = "    <a href=\"%s\"%s>%s</a>\n"
  60         InternalFlag = ".internal"
  61         GPGSigExt    = ".asc"
  62
  63         Warranty = `This program is free software: you can redistribute it and/or modify
  64 it under the terms of the GNU General Public License as published by
  65 the Free Software Foundation, version 3 of the License.
  66
  67 This program is distributed in the hope that it will be useful,
  68 but WITHOUT ANY WARRANTY; without even the implied warranty of
  69 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  70 GNU General Public License for more details.
  71
  72 You should have received a copy of the GNU General Public License
  73 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  74 )
  75
  76 const (
  77         HashAlgoSHA256     = "sha256"
  78         HashAlgoBLAKE2b256 = "blake2_256"
  79         HashAlgoSHA512     = "sha512"
  80         HashAlgoMD5        = "md5"
  81 )
  82
  83 var (
  84         pkgPyPI         = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  85         normalizationRe = regexp.MustCompilePOSIX("[-_.]+")
  86
  87         knownHashAlgos []string = []string{
  88                 HashAlgoSHA256,
  89                 HashAlgoBLAKE2b256,
  90                 HashAlgoSHA512,
  91                 HashAlgoMD5,
  92         }
  93
  94         root       = flag.String("root", "./packages", "Path to packages directory")
  95         bind       = flag.String("bind", "[::]:8080", "Address to bind to")
  96         maxClients = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  97         doUCSPI    = flag.Bool("ucspi", false, "Work as UCSPI-TCP service")
  98
  99         tlsCert = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
 100         tlsKey  = flag.String("tls-key", "", "Path to TLS X.509 private key")
 101
 102         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
 103         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
 104         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
 105
 106         pypiURL      = flag.String("pypi", "https://pypi.org/simple/", "Upstream (PyPI) URL")
 107         pypiCertHash = flag.String("pypi-cert-hash", "", "Authenticate upstream by its X.509 certificate's SPKI SHA256 hash")
 108
 109         passwdPath     = flag.String("passwd", "", "Path to FIFO for upload authentication")
 110         passwdListPath = flag.String("passwd-list", "", "Path to FIFO for login listing")
 111         passwdCheck    = flag.Bool("passwd-check", false, "Run password checker")
 112
 113         logTimestamped = flag.Bool("log-timestamped", false, "Prepend timestmap to log messages")
 114         fsck           = flag.Bool("fsck", false, "Check integrity of all packages (errors are in stderr)")
 115         version        = flag.Bool("version", false, "Print version information")
 116         warranty       = flag.Bool("warranty", false, "Print warranty information")
 117
 118         killed        bool
 119         pypiURLParsed *url.URL
 120 )
 121
 122 func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool {
 123         path := filepath.Join(*root, pkgName)
 124         if _, err := os.Stat(path); os.IsNotExist(err) {
 125                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
 126                         log.Println("error", r.RemoteAddr, "mkdir", pkgName, err)
 127                         http.Error(w, err.Error(), http.StatusInternalServerError)
 128                         return false
 129                 }
 130                 log.Println(r.RemoteAddr, "mkdir", pkgName)
 131         }
 132         return true
 133 }
 134
 135 func listRoot(w http.ResponseWriter, r *http.Request) {
 136         files, err := ioutil.ReadDir(*root)
 137         if err != nil {
 138                 log.Println("error", r.RemoteAddr, "root", err)
 139                 http.Error(w, err.Error(), http.StatusInternalServerError)
 140                 return
 141         }
 142         var result bytes.Buffer
 143         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
 144         for _, file := range files {
 145                 if file.Mode().IsDir() {
 146                         result.WriteString(fmt.Sprintf(
 147                                 HTMLElement,
 148                                 *refreshURLPath+file.Name()+"/",
 149                                 "", file.Name(),
 150                         ))
 151                 }
 152         }
 153         result.WriteString(HTMLEnd)
 154         w.Write(result.Bytes())
 155 }
 156
 157 func listDir(
 158         w http.ResponseWriter,
 159         r *http.Request,
 160         pkgName string,
 161         autorefresh, gpgUpdate bool,
 162 ) {
 163         dirPath := filepath.Join(*root, pkgName)
 164         if autorefresh {
 165                 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
 166                         return
 167                 }
 168         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "", false) {
 169                 return
 170         }
 171         fis, err := ioutil.ReadDir(dirPath)
 172         if err != nil {
 173                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 174                 http.Error(w, err.Error(), http.StatusInternalServerError)
 175                 return
 176         }
 177         files := make(map[string]struct{}, len(fis)/2)
 178         for _, fi := range fis {
 179                 files[fi.Name()] = struct{}{}
 180         }
 181         var result bytes.Buffer
 182         result.WriteString(fmt.Sprintf(HTMLBegin, pkgName))
 183         for _, algo := range knownHashAlgos {
 184                 for fn := range files {
 185                         if killed {
 186                                 // Skip expensive I/O when shutting down
 187                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 188                                 return
 189                         }
 190                         if !strings.HasSuffix(fn, "."+algo) {
 191                                 continue
 192                         }
 193                         delete(files, fn)
 194                         digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
 195                         if err != nil {
 196                                 log.Println("error", r.RemoteAddr, "list", fn, err)
 197                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 198                                 return
 199                         }
 200                         fnClean := strings.TrimSuffix(fn, "."+algo)
 201                         delete(files, fnClean)
 202                         gpgSigAttr := ""
 203                         if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil {
 204                                 gpgSigAttr = " data-gpg-sig=true"
 205                                 delete(files, fnClean+GPGSigExt)
 206                         }
 207                         result.WriteString(fmt.Sprintf(
 208                                 HTMLElement,
 209                                 strings.Join([]string{
 210                                         *refreshURLPath, pkgName, "/", fnClean,
 211                                         "#", algo, "=", hex.EncodeToString(digest),
 212                                 }, ""),
 213                                 gpgSigAttr,
 214                                 fnClean,
 215                         ))
 216                 }
 217         }
 218         result.WriteString(HTMLEnd)
 219         w.Write(result.Bytes())
 220 }
 221
 222 func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) {
 223         log.Println(r.RemoteAddr, "get", filename)
 224         path := filepath.Join(*root, pkgName, filename)
 225         if _, err := os.Stat(path); os.IsNotExist(err) {
 226                 if !refreshDir(w, r, pkgName, filename, false) {
 227                         return
 228                 }
 229         }
 230         http.ServeFile(w, r, path)
 231 }
 232
 233 func handler(w http.ResponseWriter, r *http.Request) {
 234         switch r.Method {
 235         case "GET":
 236                 var path string
 237                 var autorefresh bool
 238                 var gpgUpdate bool
 239                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 240                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 241                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
 242                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 243                         autorefresh = true
 244                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
 245                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
 246                         autorefresh = true
 247                         gpgUpdate = true
 248                 } else {
 249                         http.Error(w, "unknown action", http.StatusBadRequest)
 250                         return
 251                 }
 252                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 253                 if len(parts) > 2 {
 254                         http.Error(w, "invalid path", http.StatusBadRequest)
 255                         return
 256                 }
 257                 if len(parts) == 1 {
 258                         if parts[0] == "" {
 259                                 listRoot(w, r)
 260                         } else {
 261                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
 262                         }
 263                 } else {
 264                         servePkg(w, r, parts[0], parts[1])
 265                 }
 266         case "POST":
 267                 serveUpload(w, r)
 268         default:
 269                 http.Error(w, "unknown action", http.StatusBadRequest)
 270         }
 271 }
 272
 273 func main() {
 274         flag.Parse()
 275         if *warranty {
 276                 fmt.Println(Warranty)
 277                 return
 278         }
 279         if *version {
 280                 fmt.Println("GoCheese", Version, "built with", runtime.Version())
 281                 return
 282         }
 283
 284         if *logTimestamped {
 285                 log.SetFlags(log.Ldate | log.Lmicroseconds | log.Lshortfile)
 286         } else {
 287                 log.SetFlags(log.Lshortfile)
 288         }
 289         if !*doUCSPI {
 290                 log.SetOutput(os.Stdout)
 291         }
 292
 293         if *fsck {
 294                 if !goodIntegrity() {
 295                         os.Exit(1)
 296                 }
 297                 return
 298         }
 299
 300         if *passwdCheck {
 301                 if passwdReader(os.Stdin) {
 302                         os.Exit(0)
 303                 } else {
 304                         os.Exit(1)
 305                 }
 306         }
 307
 308         if *passwdPath != "" {
 309                 go func() {
 310                         for {
 311                                 fd, err := os.OpenFile(
 312                                         *passwdPath,
 313                                         os.O_RDONLY,
 314                                         os.FileMode(0666),
 315                                 )
 316                                 if err != nil {
 317                                         log.Fatalln(err)
 318                                 }
 319                                 passwdReader(fd)
 320                                 fd.Close()
 321                         }
 322                 }()
 323         }
 324         if *passwdListPath != "" {
 325                 go func() {
 326                         for {
 327                                 fd, err := os.OpenFile(
 328                                         *passwdListPath,
 329                                         os.O_WRONLY|os.O_APPEND,
 330                                         os.FileMode(0666),
 331                                 )
 332                                 if err != nil {
 333                                         log.Fatalln(err)
 334                                 }
 335                                 passwdLister(fd)
 336                                 fd.Close()
 337                         }
 338                 }()
 339         }
 340
 341         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 342                 log.Fatalln("Both -tls-cert and -tls-key are required")
 343         }
 344
 345         var err error
 346         pypiURLParsed, err = url.Parse(*pypiURL)
 347         if err != nil {
 348                 log.Fatalln(err)
 349         }
 350         tlsConfig := tls.Config{
 351                 ClientSessionCache: tls.NewLRUClientSessionCache(16),
 352                 NextProtos:         []string{"h2", "http/1.1"},
 353         }
 354         pypiHTTPTransport = http.Transport{
 355                 ForceAttemptHTTP2: true,
 356                 TLSClientConfig:   &tlsConfig,
 357         }
 358         if *pypiCertHash != "" {
 359                 ourDgst, err := hex.DecodeString(*pypiCertHash)
 360                 if err != nil {
 361                         log.Fatalln(err)
 362                 }
 363                 tlsConfig.VerifyConnection = func(s tls.ConnectionState) error {
 364                         spki := s.VerifiedChains[0][0].RawSubjectPublicKeyInfo
 365                         theirDgst := sha256.Sum256(spki)
 366                         if bytes.Compare(ourDgst, theirDgst[:]) != 0 {
 367                                 return errors.New("certificate's SPKI digest mismatch")
 368                         }
 369                         return nil
 370                 }
 371         }
 372
 373         server := &http.Server{
 374                 ReadTimeout:  time.Minute,
 375                 WriteTimeout: time.Minute,
 376         }
 377         http.HandleFunc(*norefreshURLPath, handler)
 378         http.HandleFunc(*refreshURLPath, handler)
 379         if *gpgUpdateURLPath != "" {
 380                 http.HandleFunc(*gpgUpdateURLPath, handler)
 381         }
 382
 383         if *doUCSPI {
 384                 server.SetKeepAlivesEnabled(false)
 385                 ln := &UCSPI{}
 386                 server.ConnState = connStater
 387                 err := server.Serve(ln)
 388                 if _, ok := err.(UCSPIAlreadyAccepted); !ok {
 389                         log.Fatalln(err)
 390                 }
 391                 UCSPIJob.Wait()
 392                 return
 393         }
 394
 395         ln, err := net.Listen("tcp", *bind)
 396         if err != nil {
 397                 log.Fatal(err)
 398         }
 399         ln = netutil.LimitListener(ln, *maxClients)
 400
 401         needsShutdown := make(chan os.Signal, 0)
 402         exitErr := make(chan error, 0)
 403         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 404         go func(s *http.Server) {
 405                 <-needsShutdown
 406                 killed = true
 407                 log.Println("shutting down")
 408                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 409                 exitErr <- s.Shutdown(ctx)
 410                 cancel()
 411         }(server)
 412
 413         log.Println(
 414                 "GoCheese", Version, "listens:",
 415                 "root:", *root,
 416                 "bind:", *bind,
 417                 "pypi:", *pypiURL,
 418         )
 419         if *tlsCert == "" {
 420                 err = server.Serve(ln)
 421         } else {
 422                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 423         }
 424         if err != http.ErrServerClosed {
 425                 log.Fatal(err)
 426         }
 427         if err := <-exitErr; err != nil {
 428                 log.Fatal(err)
 429         }
 430 }