main.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
   4               2019-2021 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "crypto/tls"
  27         "encoding/hex"
  28         "errors"
  29         "flag"
  30         "fmt"
  31         "io/ioutil"
  32         "log"
  33         "net"
  34         "net/http"
  35         "net/url"
  36         "os"
  37         "os/signal"
  38         "path/filepath"
  39         "regexp"
  40         "runtime"
  41         "strings"
  42         "syscall"
  43         "time"
  44
  45         "golang.org/x/net/netutil"
  46 )
  47
  48 const (
  49         Version   = "3.0.0"
  50         UserAgent = "GoCheese/" + Version
  51         HTMLBegin = `<!DOCTYPE html>
  52 <html>
  53   <head>
  54     <meta name="pypi:repository-version" content="1.0">
  55     <title>Links for %s</title>
  56   </head>
  57   <body>
  58 `
  59         HTMLEnd      = "  </body>\n</html>\n"
  60         HTMLElement  = "    <a href=\"%s\"%s>%s</a>\n"
  61         InternalFlag = ".internal"
  62         GPGSigExt    = ".asc"
  63
  64         Warranty = `This program is free software: you can redistribute it and/or modify
  65 it under the terms of the GNU General Public License as published by
  66 the Free Software Foundation, version 3 of the License.
  67
  68 This program is distributed in the hope that it will be useful,
  69 but WITHOUT ANY WARRANTY; without even the implied warranty of
  70 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  71 GNU General Public License for more details.
  72
  73 You should have received a copy of the GNU General Public License
  74 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  75 )
  76
  77 const (
  78         HashAlgoSHA256     = "sha256"
  79         HashAlgoBLAKE2b256 = "blake2_256"
  80         HashAlgoSHA512     = "sha512"
  81         HashAlgoMD5        = "md5"
  82 )
  83
  84 var (
  85         pkgPyPI         = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  86         normalizationRe = regexp.MustCompilePOSIX("[-_.]+")
  87
  88         knownHashAlgos []string = []string{
  89                 HashAlgoSHA256,
  90                 HashAlgoBLAKE2b256,
  91                 HashAlgoSHA512,
  92                 HashAlgoMD5,
  93         }
  94
  95         root       = flag.String("root", "./packages", "Path to packages directory")
  96         bind       = flag.String("bind", "[::]:8080", "Address to bind to")
  97         maxClients = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  98         doUCSPI    = flag.Bool("ucspi", false, "Work as UCSPI-TCP service")
  99
 100         tlsCert = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
 101         tlsKey  = flag.String("tls-key", "", "Path to TLS X.509 private key")
 102
 103         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
 104         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
 105         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
 106
 107         pypiURL      = flag.String("pypi", "https://pypi.org/simple/", "Upstream (PyPI) URL")
 108         pypiCertHash = flag.String("pypi-cert-hash", "", "Authenticate upstream by its X.509 certificate's SPKI SHA256 hash")
 109
 110         passwdPath     = flag.String("passwd", "", "Path to FIFO for upload authentication")
 111         passwdListPath = flag.String("passwd-list", "", "Path to FIFO for login listing")
 112         passwdCheck    = flag.Bool("passwd-check", false, "Run password checker")
 113
 114         logTimestamped = flag.Bool("log-timestamped", false, "Prepend timestmap to log messages")
 115         fsck           = flag.Bool("fsck", false, "Check integrity of all packages (errors are in stderr)")
 116         version        = flag.Bool("version", false, "Print version information")
 117         warranty       = flag.Bool("warranty", false, "Print warranty information")
 118
 119         killed        bool
 120         pypiURLParsed *url.URL
 121 )
 122
 123 func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool {
 124         path := filepath.Join(*root, pkgName)
 125         if _, err := os.Stat(path); os.IsNotExist(err) {
 126                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
 127                         log.Println("error", r.RemoteAddr, "mkdir", pkgName, err)
 128                         http.Error(w, err.Error(), http.StatusInternalServerError)
 129                         return false
 130                 }
 131                 log.Println(r.RemoteAddr, "mkdir", pkgName)
 132         }
 133         return true
 134 }
 135
 136 func listRoot(w http.ResponseWriter, r *http.Request) {
 137         files, err := ioutil.ReadDir(*root)
 138         if err != nil {
 139                 log.Println("error", r.RemoteAddr, "root", err)
 140                 http.Error(w, err.Error(), http.StatusInternalServerError)
 141                 return
 142         }
 143         var result bytes.Buffer
 144         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
 145         for _, file := range files {
 146                 if file.Mode().IsDir() {
 147                         result.WriteString(fmt.Sprintf(
 148                                 HTMLElement,
 149                                 *refreshURLPath+file.Name()+"/",
 150                                 "", file.Name(),
 151                         ))
 152                 }
 153         }
 154         result.WriteString(HTMLEnd)
 155         w.Write(result.Bytes())
 156 }
 157
 158 func listDir(
 159         w http.ResponseWriter,
 160         r *http.Request,
 161         pkgName string,
 162         autorefresh, gpgUpdate bool,
 163 ) {
 164         dirPath := filepath.Join(*root, pkgName)
 165         if autorefresh {
 166                 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
 167                         return
 168                 }
 169         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "", false) {
 170                 return
 171         }
 172         fis, err := ioutil.ReadDir(dirPath)
 173         if err != nil {
 174                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 175                 http.Error(w, err.Error(), http.StatusInternalServerError)
 176                 return
 177         }
 178         files := make(map[string]struct{}, len(fis)/2)
 179         for _, fi := range fis {
 180                 files[fi.Name()] = struct{}{}
 181         }
 182         var result bytes.Buffer
 183         result.WriteString(fmt.Sprintf(HTMLBegin, pkgName))
 184         for _, algo := range knownHashAlgos {
 185                 for fn := range files {
 186                         if killed {
 187                                 // Skip expensive I/O when shutting down
 188                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 189                                 return
 190                         }
 191                         if !strings.HasSuffix(fn, "."+algo) {
 192                                 continue
 193                         }
 194                         delete(files, fn)
 195                         digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
 196                         if err != nil {
 197                                 log.Println("error", r.RemoteAddr, "list", fn, err)
 198                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 199                                 return
 200                         }
 201                         fnClean := strings.TrimSuffix(fn, "."+algo)
 202                         delete(files, fnClean)
 203                         gpgSigAttr := ""
 204                         if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil {
 205                                 gpgSigAttr = " data-gpg-sig=true"
 206                                 delete(files, fnClean+GPGSigExt)
 207                         }
 208                         result.WriteString(fmt.Sprintf(
 209                                 HTMLElement,
 210                                 strings.Join([]string{
 211                                         *refreshURLPath, pkgName, "/", fnClean,
 212                                         "#", algo, "=", hex.EncodeToString(digest),
 213                                 }, ""),
 214                                 gpgSigAttr,
 215                                 fnClean,
 216                         ))
 217                 }
 218         }
 219         result.WriteString(HTMLEnd)
 220         w.Write(result.Bytes())
 221 }
 222
 223 func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) {
 224         log.Println(r.RemoteAddr, "get", filename)
 225         path := filepath.Join(*root, pkgName, filename)
 226         if _, err := os.Stat(path); os.IsNotExist(err) {
 227                 if !refreshDir(w, r, pkgName, filename, false) {
 228                         return
 229                 }
 230         }
 231         http.ServeFile(w, r, path)
 232 }
 233
 234 func handler(w http.ResponseWriter, r *http.Request) {
 235         w.Header().Set("Server", UserAgent)
 236         switch r.Method {
 237         case "GET":
 238                 var path string
 239                 var autorefresh bool
 240                 var gpgUpdate bool
 241                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 242                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 243                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
 244                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 245                         autorefresh = true
 246                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
 247                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
 248                         autorefresh = true
 249                         gpgUpdate = true
 250                 } else {
 251                         http.Error(w, "unknown action", http.StatusBadRequest)
 252                         return
 253                 }
 254                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 255                 if len(parts) > 2 {
 256                         http.Error(w, "invalid path", http.StatusBadRequest)
 257                         return
 258                 }
 259                 if len(parts) == 1 {
 260                         if parts[0] == "" {
 261                                 listRoot(w, r)
 262                         } else {
 263                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
 264                         }
 265                 } else {
 266                         servePkg(w, r, parts[0], parts[1])
 267                 }
 268         case "POST":
 269                 serveUpload(w, r)
 270         default:
 271                 http.Error(w, "unknown action", http.StatusBadRequest)
 272         }
 273 }
 274
 275 func main() {
 276         flag.Parse()
 277         if *warranty {
 278                 fmt.Println(Warranty)
 279                 return
 280         }
 281         if *version {
 282                 fmt.Println("GoCheese", Version, "built with", runtime.Version())
 283                 return
 284         }
 285
 286         if *logTimestamped {
 287                 log.SetFlags(log.Ldate | log.Lmicroseconds | log.Lshortfile)
 288         } else {
 289                 log.SetFlags(log.Lshortfile)
 290         }
 291         if !*doUCSPI {
 292                 log.SetOutput(os.Stdout)
 293         }
 294
 295         if *fsck {
 296                 if !goodIntegrity() {
 297                         os.Exit(1)
 298                 }
 299                 return
 300         }
 301
 302         if *passwdCheck {
 303                 if passwdReader(os.Stdin) {
 304                         os.Exit(0)
 305                 } else {
 306                         os.Exit(1)
 307                 }
 308         }
 309
 310         if *passwdPath != "" {
 311                 go func() {
 312                         for {
 313                                 fd, err := os.OpenFile(
 314                                         *passwdPath,
 315                                         os.O_RDONLY,
 316                                         os.FileMode(0666),
 317                                 )
 318                                 if err != nil {
 319                                         log.Fatalln(err)
 320                                 }
 321                                 passwdReader(fd)
 322                                 fd.Close()
 323                         }
 324                 }()
 325         }
 326         if *passwdListPath != "" {
 327                 go func() {
 328                         for {
 329                                 fd, err := os.OpenFile(
 330                                         *passwdListPath,
 331                                         os.O_WRONLY|os.O_APPEND,
 332                                         os.FileMode(0666),
 333                                 )
 334                                 if err != nil {
 335                                         log.Fatalln(err)
 336                                 }
 337                                 passwdLister(fd)
 338                                 fd.Close()
 339                         }
 340                 }()
 341         }
 342
 343         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 344                 log.Fatalln("Both -tls-cert and -tls-key are required")
 345         }
 346
 347         var err error
 348         pypiURLParsed, err = url.Parse(*pypiURL)
 349         if err != nil {
 350                 log.Fatalln(err)
 351         }
 352         tlsConfig := tls.Config{
 353                 ClientSessionCache: tls.NewLRUClientSessionCache(16),
 354                 NextProtos:         []string{"h2", "http/1.1"},
 355         }
 356         pypiHTTPTransport = http.Transport{
 357                 ForceAttemptHTTP2: true,
 358                 TLSClientConfig:   &tlsConfig,
 359         }
 360         if *pypiCertHash != "" {
 361                 ourDgst, err := hex.DecodeString(*pypiCertHash)
 362                 if err != nil {
 363                         log.Fatalln(err)
 364                 }
 365                 tlsConfig.VerifyConnection = func(s tls.ConnectionState) error {
 366                         spki := s.VerifiedChains[0][0].RawSubjectPublicKeyInfo
 367                         theirDgst := sha256.Sum256(spki)
 368                         if bytes.Compare(ourDgst, theirDgst[:]) != 0 {
 369                                 return errors.New("certificate's SPKI digest mismatch")
 370                         }
 371                         return nil
 372                 }
 373         }
 374
 375         server := &http.Server{
 376                 ReadTimeout:  time.Minute,
 377                 WriteTimeout: time.Minute,
 378         }
 379         http.HandleFunc(*norefreshURLPath, handler)
 380         http.HandleFunc(*refreshURLPath, handler)
 381         if *gpgUpdateURLPath != "" {
 382                 http.HandleFunc(*gpgUpdateURLPath, handler)
 383         }
 384
 385         if *doUCSPI {
 386                 server.SetKeepAlivesEnabled(false)
 387                 ln := &UCSPI{}
 388                 server.ConnState = connStater
 389                 err := server.Serve(ln)
 390                 if _, ok := err.(UCSPIAlreadyAccepted); !ok {
 391                         log.Fatalln(err)
 392                 }
 393                 UCSPIJob.Wait()
 394                 return
 395         }
 396
 397         ln, err := net.Listen("tcp", *bind)
 398         if err != nil {
 399                 log.Fatal(err)
 400         }
 401         ln = netutil.LimitListener(ln, *maxClients)
 402
 403         needsShutdown := make(chan os.Signal, 0)
 404         exitErr := make(chan error, 0)
 405         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 406         go func(s *http.Server) {
 407                 <-needsShutdown
 408                 killed = true
 409                 log.Println("shutting down")
 410                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 411                 exitErr <- s.Shutdown(ctx)
 412                 cancel()
 413         }(server)
 414
 415         log.Println(
 416                 "GoCheese", Version, "listens:",
 417                 "root:", *root,
 418                 "bind:", *bind,
 419                 "pypi:", *pypiURL,
 420         )
 421         if *tlsCert == "" {
 422                 err = server.Serve(ln)
 423         } else {
 424                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 425         }
 426         if err != http.ErrServerClosed {
 427                 log.Fatal(err)
 428         }
 429         if err := <-exitErr; err != nil {
 430                 log.Fatal(err)
 431         }
 432 }