gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "encoding/hex"
  27         "flag"
  28         "fmt"
  29         "io"
  30         "io/ioutil"
  31         "log"
  32         "net"
  33         "net/http"
  34         "net/url"
  35         "os"
  36         "os/signal"
  37         "path/filepath"
  38         "regexp"
  39         "runtime"
  40         "strings"
  41         "syscall"
  42         "time"
  43
  44         "golang.org/x/net/netutil"
  45 )
  46
  47 const (
  48         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  49         HTMLEnd      = "</body></html>"
  50         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  51         SHA256Prefix = "sha256="
  52         SHA256Ext    = ".sha256"
  53         InternalFlag = ".internal"
  54
  55         Warranty = `This program is free software: you can redistribute it and/or modify
  56 it under the terms of the GNU General Public License as published by
  57 the Free Software Foundation, version 3 of the License.
  58
  59 This program is distributed in the hope that it will be useful,
  60 but WITHOUT ANY WARRANTY; without even the implied warranty of
  61 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  62 GNU General Public License for more details.
  63
  64 You should have received a copy of the GNU General Public License
  65 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  66 )
  67
  68 var (
  69         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  70         Version string = "UNKNOWN"
  71
  72         root             = flag.String("root", "./packages", "Path to packages directory")
  73         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  74         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
  75         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
  76         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  77         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  78         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  79         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  80         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  81         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  82         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  83         version          = flag.Bool("version", false, "Print version information")
  84         warranty         = flag.Bool("warranty", false, "Print warranty information")
  85
  86         killed bool
  87 )
  88
  89 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  90         path := filepath.Join(*root, dir)
  91         if _, err := os.Stat(path); os.IsNotExist(err) {
  92                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
  93                         http.Error(w, err.Error(), http.StatusInternalServerError)
  94                         return false
  95                 }
  96                 log.Println(r.RemoteAddr, "mkdir", dir)
  97         }
  98         return true
  99 }
 100
 101 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
 102         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
 103                 return true
 104         }
 105         resp, err := http.Get(*pypiURL + dir + "/")
 106         if err != nil {
 107                 http.Error(w, err.Error(), http.StatusBadGateway)
 108                 return false
 109         }
 110         body, err := ioutil.ReadAll(resp.Body)
 111         resp.Body.Close()
 112         if err != nil {
 113                 http.Error(w, err.Error(), http.StatusBadGateway)
 114                 return false
 115         }
 116         if !mkdirForPkg(w, r, dir) {
 117                 return false
 118         }
 119         var submatches []string
 120         var uri string
 121         var filename string
 122         var path string
 123         var pkgURL *url.URL
 124         var digest []byte
 125         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 126                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 127                 if len(submatches) == 0 {
 128                         continue
 129                 }
 130                 uri = submatches[1]
 131                 filename = submatches[2]
 132                 if pkgURL, err = url.Parse(uri); err != nil {
 133                         http.Error(w, err.Error(), http.StatusInternalServerError)
 134                         return false
 135                 }
 136                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 137                 if err != nil {
 138                         http.Error(w, err.Error(), http.StatusBadGateway)
 139                         return false
 140                 }
 141                 if filename == filenameGet {
 142                         if killed {
 143                                 // Skip heavy remote call, when shutting down
 144                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 145                                 return false
 146                         }
 147                         log.Println(r.RemoteAddr, "pypi download", filename)
 148                         path = filepath.Join(*root, dir, filename)
 149                         resp, err = http.Get(uri)
 150                         if err != nil {
 151                                 http.Error(w, err.Error(), http.StatusBadGateway)
 152                                 return false
 153                         }
 154                         defer resp.Body.Close()
 155                         hasher := sha256.New()
 156                         dst, err := TempFile(filepath.Join(*root, dir))
 157                         if err != nil {
 158                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 159                                 return false
 160                         }
 161                         wr := io.MultiWriter(hasher, dst)
 162                         if _, err = io.Copy(wr, resp.Body); err != nil {
 163                                 os.Remove(dst.Name())
 164                                 dst.Close()
 165                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 166                                 return false
 167                         }
 168                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 169                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 170                                 os.Remove(dst.Name())
 171                                 dst.Close()
 172                                 http.Error(w, err.Error(), http.StatusBadGateway)
 173                                 return false
 174                         }
 175                         if err = dst.Sync(); err != nil {
 176                                 os.Remove(dst.Name())
 177                                 dst.Close()
 178                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 179                                 return false
 180                         }
 181                         dst.Close()
 182                         if err = os.Rename(dst.Name(), path); err != nil {
 183                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 184                                 return false
 185                         }
 186                 }
 187                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 188                 _, err = os.Stat(path)
 189                 if err == nil {
 190                         continue
 191                 }
 192                 if !os.IsNotExist(err) {
 193                         http.Error(w, err.Error(), http.StatusInternalServerError)
 194                         return false
 195                 }
 196                 log.Println(r.RemoteAddr, "pypi touch", filename)
 197                 if err = ioutil.WriteFile(path, digest, os.FileMode(0666)); err != nil {
 198                         http.Error(w, err.Error(), http.StatusInternalServerError)
 199                         return false
 200                 }
 201         }
 202         return true
 203 }
 204
 205 func listRoot(w http.ResponseWriter, r *http.Request) {
 206         files, err := ioutil.ReadDir(*root)
 207         if err != nil {
 208                 http.Error(w, err.Error(), http.StatusInternalServerError)
 209                 return
 210         }
 211         var result bytes.Buffer
 212         result.WriteString(fmt.Sprintf(HTMLBegin, "root", "root"))
 213         for _, file := range files {
 214                 if file.Mode().IsDir() {
 215                         result.WriteString(fmt.Sprintf(
 216                                 HTMLElement,
 217                                 *refreshURLPath+file.Name()+"/",
 218                                 file.Name(),
 219                         ))
 220                 }
 221         }
 222         result.WriteString(HTMLEnd)
 223         w.Write(result.Bytes())
 224 }
 225
 226 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 227         dirPath := filepath.Join(*root, dir)
 228         if autorefresh {
 229                 if !refreshDir(w, r, dir, "") {
 230                         return
 231                 }
 232         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 233                 return
 234         }
 235         files, err := ioutil.ReadDir(dirPath)
 236         if err != nil {
 237                 http.Error(w, err.Error(), http.StatusInternalServerError)
 238                 return
 239         }
 240         var result bytes.Buffer
 241         result.WriteString(fmt.Sprintf(HTMLBegin, dir, dir))
 242         var data []byte
 243         var filenameClean string
 244         for _, file := range files {
 245                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 246                         continue
 247                 }
 248                 if killed {
 249                         // Skip expensive I/O when shutting down
 250                         http.Error(w, "shutting down", http.StatusInternalServerError)
 251                         return
 252                 }
 253                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 254                 if err != nil {
 255                         http.Error(w, err.Error(), http.StatusInternalServerError)
 256                         return
 257                 }
 258                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 259                 result.WriteString(fmt.Sprintf(
 260                         HTMLElement,
 261                         strings.Join([]string{
 262                                 *refreshURLPath, dir, "/",
 263                                 filenameClean, "#", SHA256Prefix, hex.EncodeToString(data),
 264                         }, ""),
 265                         filenameClean,
 266                 ))
 267         }
 268         result.WriteString(HTMLEnd)
 269         w.Write(result.Bytes())
 270 }
 271
 272 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 273         log.Println(r.RemoteAddr, "get", filename)
 274         path := filepath.Join(*root, dir, filename)
 275         if _, err := os.Stat(path); os.IsNotExist(err) {
 276                 if !refreshDir(w, r, dir, filename) {
 277                         return
 278                 }
 279         }
 280         http.ServeFile(w, r, path)
 281 }
 282
 283 func serveUpload(w http.ResponseWriter, r *http.Request) {
 284         username, password, ok := r.BasicAuth()
 285         if !ok {
 286                 log.Println(r.RemoteAddr, "unauthenticated", username)
 287                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 288                 return
 289         }
 290         auther, ok := passwords[username]
 291         if !ok || !auther.Auth(password) {
 292                 log.Println(r.RemoteAddr, "unauthenticated", username)
 293                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 294                 return
 295         }
 296         var err error
 297         if err = r.ParseMultipartForm(1 << 20); err != nil {
 298                 http.Error(w, err.Error(), http.StatusBadRequest)
 299                 return
 300         }
 301         var digestExpected []byte
 302         if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists {
 303                 digestExpected, err = hex.DecodeString(digestExpectedHex[0])
 304                 if err != nil {
 305                         http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest)
 306                         return
 307                 }
 308         }
 309         for _, file := range r.MultipartForm.File["content"] {
 310                 filename := file.Filename
 311                 log.Println(r.RemoteAddr, "put", filename, "by", username)
 312                 dir := filename[:strings.LastIndex(filename, "-")]
 313                 dirPath := filepath.Join(*root, dir)
 314                 path := filepath.Join(dirPath, filename)
 315                 if _, err = os.Stat(path); err == nil {
 316                         log.Println(r.RemoteAddr, "already exists", filename)
 317                         http.Error(w, "Already exists", http.StatusBadRequest)
 318                         return
 319                 }
 320                 if !mkdirForPkg(w, r, dir) {
 321                         return
 322                 }
 323                 internalPath := filepath.Join(dirPath, InternalFlag)
 324                 var dst *os.File
 325                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 326                         if dst, err = os.Create(internalPath); err != nil {
 327                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 328                                 return
 329                         }
 330                         dst.Close()
 331                 }
 332                 src, err := file.Open()
 333                 defer src.Close()
 334                 if err != nil {
 335                         http.Error(w, err.Error(), http.StatusInternalServerError)
 336                         return
 337                 }
 338                 dst, err = TempFile(dirPath)
 339                 if err != nil {
 340                         http.Error(w, err.Error(), http.StatusInternalServerError)
 341                         return
 342                 }
 343                 hasher := sha256.New()
 344                 wr := io.MultiWriter(hasher, dst)
 345                 if _, err = io.Copy(wr, src); err != nil {
 346                         os.Remove(dst.Name())
 347                         dst.Close()
 348                         http.Error(w, err.Error(), http.StatusInternalServerError)
 349                         return
 350                 }
 351                 if err = dst.Sync(); err != nil {
 352                         os.Remove(dst.Name())
 353                         dst.Close()
 354                         http.Error(w, err.Error(), http.StatusInternalServerError)
 355                         return
 356                 }
 357                 dst.Close()
 358                 digest := hasher.Sum(nil)
 359                 if digestExpected != nil {
 360                         if bytes.Compare(digestExpected, digest) == 0 {
 361                                 log.Println(r.RemoteAddr, filename, "good checksum received")
 362                         } else {
 363                                 log.Println(r.RemoteAddr, filename, "bad checksum received")
 364                                 http.Error(w, "bad checksum", http.StatusBadRequest)
 365                                 os.Remove(dst.Name())
 366                                 return
 367                         }
 368                 }
 369                 if err = os.Rename(dst.Name(), path); err != nil {
 370                         http.Error(w, err.Error(), http.StatusInternalServerError)
 371                         return
 372                 }
 373                 if err = ioutil.WriteFile(path+SHA256Ext, digest, os.FileMode(0666)); err != nil {
 374                         http.Error(w, err.Error(), http.StatusInternalServerError)
 375                         return
 376                 }
 377         }
 378 }
 379
 380 func handler(w http.ResponseWriter, r *http.Request) {
 381         switch r.Method {
 382         case "GET":
 383                 var path string
 384                 var autorefresh bool
 385                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 386                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 387                         autorefresh = false
 388                 } else {
 389                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 390                         autorefresh = true
 391                 }
 392                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 393                 if len(parts) > 2 {
 394                         http.Error(w, "invalid path", http.StatusBadRequest)
 395                         return
 396                 }
 397                 if len(parts) == 1 {
 398                         if parts[0] == "" {
 399                                 listRoot(w, r)
 400                         } else {
 401                                 listDir(w, r, parts[0], autorefresh)
 402                         }
 403                 } else {
 404                         servePkg(w, r, parts[0], parts[1])
 405                 }
 406         case "POST":
 407                 serveUpload(w, r)
 408         default:
 409                 http.Error(w, "unknown action", http.StatusBadRequest)
 410         }
 411 }
 412
 413 func goodIntegrity() bool {
 414         dirs, err := ioutil.ReadDir(*root)
 415         if err != nil {
 416                 log.Fatal(err)
 417         }
 418         hasher := sha256.New()
 419         digest := make([]byte, sha256.Size)
 420         isGood := true
 421         var data []byte
 422         var pkgName string
 423         for _, dir := range dirs {
 424                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 425                 if err != nil {
 426                         log.Fatal(err)
 427                 }
 428                 for _, file := range files {
 429                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 430                                 continue
 431                         }
 432                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 433                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 434                         if err != nil {
 435                                 if os.IsNotExist(err) {
 436                                         continue
 437                                 }
 438                                 log.Fatal(err)
 439                         }
 440                         hasher.Write(data)
 441                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 442                         if err != nil {
 443                                 log.Fatal(err)
 444                         }
 445                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 446                                 log.Println(pkgName, "GOOD")
 447                         } else {
 448                                 isGood = false
 449                                 log.Println(pkgName, "BAD")
 450                         }
 451                         hasher.Reset()
 452                 }
 453         }
 454         return isGood
 455 }
 456
 457 func main() {
 458         flag.Parse()
 459         if *warranty {
 460                 fmt.Println(Warranty)
 461                 return
 462         }
 463         if *version {
 464                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 465                 return
 466         }
 467         if *fsck {
 468                 if !goodIntegrity() {
 469                         os.Exit(1)
 470                 }
 471                 return
 472         }
 473         if *passwdCheck {
 474                 refreshPasswd()
 475                 return
 476         }
 477         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 478                 log.Fatalln("Both -tls-cert and -tls-key are required")
 479         }
 480         refreshPasswd()
 481         log.Println("root:", *root, "bind:", *bind)
 482
 483         ln, err := net.Listen("tcp", *bind)
 484         if err != nil {
 485                 log.Fatal(err)
 486         }
 487         ln = netutil.LimitListener(ln, *maxClients)
 488         server := &http.Server{
 489                 ReadTimeout:  time.Minute,
 490                 WriteTimeout: time.Minute,
 491         }
 492         http.HandleFunc(*norefreshURLPath, handler)
 493         http.HandleFunc(*refreshURLPath, handler)
 494
 495         needsRefreshPasswd := make(chan os.Signal, 0)
 496         needsShutdown := make(chan os.Signal, 0)
 497         exitErr := make(chan error, 0)
 498         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 499         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 500         go func() {
 501                 for range needsRefreshPasswd {
 502                         log.Println("Refreshing passwords")
 503                         refreshPasswd()
 504                 }
 505         }()
 506         go func(s *http.Server) {
 507                 <-needsShutdown
 508                 killed = true
 509                 log.Println("Shutting down")
 510                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 511                 exitErr <- s.Shutdown(ctx)
 512                 cancel()
 513         }(server)
 514
 515         if *tlsCert == "" {
 516                 err = server.Serve(ln)
 517         } else {
 518                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 519         }
 520         if err != http.ErrServerClosed {
 521                 log.Fatal(err)
 522         }
 523         if err := <-exitErr; err != nil {
 524                 log.Fatal(err)
 525         }
 526 }