gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "encoding/hex"
  27         "flag"
  28         "fmt"
  29         "io"
  30         "io/ioutil"
  31         "log"
  32         "net"
  33         "net/http"
  34         "net/url"
  35         "os"
  36         "os/signal"
  37         "path/filepath"
  38         "regexp"
  39         "runtime"
  40         "strings"
  41         "syscall"
  42         "time"
  43
  44         "golang.org/x/net/netutil"
  45 )
  46
  47 const (
  48         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  49         HTMLEnd      = "</body></html>"
  50         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  51         SHA256Prefix = "sha256="
  52         SHA256Ext    = ".sha256"
  53         InternalFlag = ".internal"
  54
  55         Warranty = `This program is free software: you can redistribute it and/or modify
  56 it under the terms of the GNU General Public License as published by
  57 the Free Software Foundation, version 3 of the License.
  58
  59 This program is distributed in the hope that it will be useful,
  60 but WITHOUT ANY WARRANTY; without even the implied warranty of
  61 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  62 GNU General Public License for more details.
  63
  64 You should have received a copy of the GNU General Public License
  65 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  66 )
  67
  68 var (
  69         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  70         Version string = "UNKNOWN"
  71
  72         root             = flag.String("root", "./packages", "Path to packages directory")
  73         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  74         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
  75         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
  76         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  77         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  78         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  79         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  80         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  81         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  82         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  83         version          = flag.Bool("version", false, "Print version information")
  84         warranty         = flag.Bool("warranty", false, "Print warranty information")
  85
  86         killed bool
  87 )
  88
  89 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  90         path := filepath.Join(*root, dir)
  91         if _, err := os.Stat(path); os.IsNotExist(err) {
  92                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
  93                         http.Error(w, err.Error(), http.StatusInternalServerError)
  94                         return false
  95                 }
  96                 log.Println(r.RemoteAddr, "mkdir", dir)
  97         }
  98         return true
  99 }
 100
 101 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
 102         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
 103                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
 104                 return true
 105         }
 106         log.Println(r.RemoteAddr, "pypi refresh", dir)
 107         resp, err := http.Get(*pypiURL + dir + "/")
 108         if err != nil {
 109                 http.Error(w, err.Error(), http.StatusBadGateway)
 110                 return false
 111         }
 112         body, err := ioutil.ReadAll(resp.Body)
 113         resp.Body.Close()
 114         if err != nil {
 115                 http.Error(w, err.Error(), http.StatusBadGateway)
 116                 return false
 117         }
 118         if !mkdirForPkg(w, r, dir) {
 119                 return false
 120         }
 121         var submatches []string
 122         var uri string
 123         var filename string
 124         var path string
 125         var pkgURL *url.URL
 126         var digest []byte
 127         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 128                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 129                 if len(submatches) == 0 {
 130                         continue
 131                 }
 132                 uri = submatches[1]
 133                 filename = submatches[2]
 134                 if pkgURL, err = url.Parse(uri); err != nil {
 135                         http.Error(w, err.Error(), http.StatusInternalServerError)
 136                         return false
 137                 }
 138                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 139                 if err != nil {
 140                         http.Error(w, err.Error(), http.StatusBadGateway)
 141                         return false
 142                 }
 143                 if filename == filenameGet {
 144                         if killed {
 145                                 // Skip heavy remote call, when shutting down
 146                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 147                                 return false
 148                         }
 149                         log.Println(r.RemoteAddr, "pypi download", filename)
 150                         path = filepath.Join(*root, dir, filename)
 151                         resp, err = http.Get(uri)
 152                         if err != nil {
 153                                 http.Error(w, err.Error(), http.StatusBadGateway)
 154                                 return false
 155                         }
 156                         defer resp.Body.Close()
 157                         hasher := sha256.New()
 158                         dst, err := TempFile(filepath.Join(*root, dir))
 159                         if err != nil {
 160                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 161                                 return false
 162                         }
 163                         wr := io.MultiWriter(hasher, dst)
 164                         if _, err = io.Copy(wr, resp.Body); err != nil {
 165                                 os.Remove(dst.Name())
 166                                 dst.Close()
 167                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 168                                 return false
 169                         }
 170                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 171                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 172                                 os.Remove(dst.Name())
 173                                 dst.Close()
 174                                 http.Error(w, err.Error(), http.StatusBadGateway)
 175                                 return false
 176                         }
 177                         if err = dst.Sync(); err != nil {
 178                                 os.Remove(dst.Name())
 179                                 dst.Close()
 180                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 181                                 return false
 182                         }
 183                         dst.Close()
 184                         if err = os.Rename(dst.Name(), path); err != nil {
 185                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 186                                 return false
 187                         }
 188                 }
 189                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 190                 _, err = os.Stat(path)
 191                 if err == nil {
 192                         continue
 193                 }
 194                 if !os.IsNotExist(err) {
 195                         http.Error(w, err.Error(), http.StatusInternalServerError)
 196                         return false
 197                 }
 198                 log.Println(r.RemoteAddr, "pypi touch", filename)
 199                 if err = ioutil.WriteFile(path, digest, os.FileMode(0666)); err != nil {
 200                         http.Error(w, err.Error(), http.StatusInternalServerError)
 201                         return false
 202                 }
 203         }
 204         return true
 205 }
 206
 207 func listRoot(w http.ResponseWriter, r *http.Request) {
 208         log.Println(r.RemoteAddr, "root")
 209         files, err := ioutil.ReadDir(*root)
 210         if err != nil {
 211                 http.Error(w, err.Error(), http.StatusInternalServerError)
 212                 return
 213         }
 214         var result bytes.Buffer
 215         result.WriteString(fmt.Sprintf(HTMLBegin, "root", "root"))
 216         for _, file := range files {
 217                 if file.Mode().IsDir() {
 218                         result.WriteString(fmt.Sprintf(
 219                                 HTMLElement,
 220                                 *refreshURLPath+file.Name()+"/",
 221                                 file.Name(),
 222                         ))
 223                 }
 224         }
 225         result.WriteString(HTMLEnd)
 226         w.Write(result.Bytes())
 227 }
 228
 229 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 230         log.Println(r.RemoteAddr, "dir", dir)
 231         dirPath := filepath.Join(*root, dir)
 232         if autorefresh {
 233                 if !refreshDir(w, r, dir, "") {
 234                         return
 235                 }
 236         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 237                 return
 238         }
 239         files, err := ioutil.ReadDir(dirPath)
 240         if err != nil {
 241                 http.Error(w, err.Error(), http.StatusInternalServerError)
 242                 return
 243         }
 244         var result bytes.Buffer
 245         result.WriteString(fmt.Sprintf(HTMLBegin, dir, dir))
 246         var data []byte
 247         var filenameClean string
 248         for _, file := range files {
 249                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 250                         continue
 251                 }
 252                 if killed {
 253                         // Skip expensive I/O when shutting down
 254                         http.Error(w, "shutting down", http.StatusInternalServerError)
 255                         return
 256                 }
 257                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 258                 if err != nil {
 259                         http.Error(w, err.Error(), http.StatusInternalServerError)
 260                         return
 261                 }
 262                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 263                 result.WriteString(fmt.Sprintf(
 264                         HTMLElement,
 265                         strings.Join([]string{
 266                                 *refreshURLPath, dir, "/",
 267                                 filenameClean, "#", SHA256Prefix, hex.EncodeToString(data),
 268                         }, ""),
 269                         filenameClean,
 270                 ))
 271         }
 272         result.WriteString(HTMLEnd)
 273         w.Write(result.Bytes())
 274 }
 275
 276 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 277         log.Println(r.RemoteAddr, "pkg", filename)
 278         path := filepath.Join(*root, dir, filename)
 279         if _, err := os.Stat(path); os.IsNotExist(err) {
 280                 if !refreshDir(w, r, dir, filename) {
 281                         return
 282                 }
 283         }
 284         http.ServeFile(w, r, path)
 285 }
 286
 287 func serveUpload(w http.ResponseWriter, r *http.Request) {
 288         username, password, ok := r.BasicAuth()
 289         if !ok {
 290                 log.Println(r.RemoteAddr, "unauthenticated", username)
 291                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 292                 return
 293         }
 294         auther, ok := passwords[username]
 295         if !ok || !auther.Auth(password) {
 296                 log.Println(r.RemoteAddr, "unauthenticated", username)
 297                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 298                 return
 299         }
 300         var err error
 301         if err = r.ParseMultipartForm(1 << 20); err != nil {
 302                 http.Error(w, err.Error(), http.StatusBadRequest)
 303                 return
 304         }
 305         for _, file := range r.MultipartForm.File["content"] {
 306                 filename := file.Filename
 307                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 308                 dir := filename[:strings.LastIndex(filename, "-")]
 309                 dirPath := filepath.Join(*root, dir)
 310                 path := filepath.Join(dirPath, filename)
 311                 if _, err = os.Stat(path); err == nil {
 312                         log.Println(r.RemoteAddr, "already exists", filename)
 313                         http.Error(w, "Already exists", http.StatusBadRequest)
 314                         return
 315                 }
 316                 if !mkdirForPkg(w, r, dir) {
 317                         return
 318                 }
 319                 internalPath := filepath.Join(dirPath, InternalFlag)
 320                 var dst *os.File
 321                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 322                         if dst, err = os.Create(internalPath); err != nil {
 323                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 324                                 return
 325                         }
 326                         dst.Close()
 327                 }
 328                 src, err := file.Open()
 329                 defer src.Close()
 330                 if err != nil {
 331                         http.Error(w, err.Error(), http.StatusInternalServerError)
 332                         return
 333                 }
 334                 dst, err = TempFile(dirPath)
 335                 if err != nil {
 336                         http.Error(w, err.Error(), http.StatusInternalServerError)
 337                         return
 338                 }
 339                 hasher := sha256.New()
 340                 wr := io.MultiWriter(hasher, dst)
 341                 if _, err = io.Copy(wr, src); err != nil {
 342                         os.Remove(dst.Name())
 343                         dst.Close()
 344                         http.Error(w, err.Error(), http.StatusInternalServerError)
 345                         return
 346                 }
 347                 if err = dst.Sync(); err != nil {
 348                         os.Remove(dst.Name())
 349                         dst.Close()
 350                         http.Error(w, err.Error(), http.StatusInternalServerError)
 351                         return
 352                 }
 353                 dst.Close()
 354                 if err = os.Rename(dst.Name(), path); err != nil {
 355                         http.Error(w, err.Error(), http.StatusInternalServerError)
 356                         return
 357                 }
 358                 if err = ioutil.WriteFile(path+SHA256Ext, hasher.Sum(nil), os.FileMode(0666)); err != nil {
 359                         http.Error(w, err.Error(), http.StatusInternalServerError)
 360                         return
 361                 }
 362         }
 363 }
 364
 365 func handler(w http.ResponseWriter, r *http.Request) {
 366         if r.Method == "GET" {
 367                 var path string
 368                 var autorefresh bool
 369                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 370                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 371                         autorefresh = false
 372                 } else {
 373                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 374                         autorefresh = true
 375                 }
 376                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 377                 if len(parts) > 2 {
 378                         http.Error(w, "invalid path", http.StatusBadRequest)
 379                         return
 380                 }
 381                 if len(parts) == 1 {
 382                         if parts[0] == "" {
 383                                 listRoot(w, r)
 384                         } else {
 385                                 listDir(w, r, parts[0], autorefresh)
 386                         }
 387                 } else {
 388                         servePkg(w, r, parts[0], parts[1])
 389                 }
 390         } else if r.Method == "POST" {
 391                 serveUpload(w, r)
 392         }
 393 }
 394
 395 func goodIntegrity() bool {
 396         dirs, err := ioutil.ReadDir(*root)
 397         if err != nil {
 398                 log.Fatal(err)
 399         }
 400         hasher := sha256.New()
 401         digest := make([]byte, sha256.Size)
 402         isGood := true
 403         var data []byte
 404         var pkgName string
 405         for _, dir := range dirs {
 406                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 407                 if err != nil {
 408                         log.Fatal(err)
 409                 }
 410                 for _, file := range files {
 411                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 412                                 continue
 413                         }
 414                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 415                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 416                         if err != nil {
 417                                 if os.IsNotExist(err) {
 418                                         continue
 419                                 }
 420                                 log.Fatal(err)
 421                         }
 422                         hasher.Write(data)
 423                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 424                         if err != nil {
 425                                 log.Fatal(err)
 426                         }
 427                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 428                                 log.Println(pkgName, "GOOD")
 429                         } else {
 430                                 isGood = false
 431                                 log.Println(pkgName, "BAD")
 432                         }
 433                         hasher.Reset()
 434                 }
 435         }
 436         return isGood
 437 }
 438
 439 func main() {
 440         flag.Parse()
 441         if *warranty {
 442                 fmt.Println(Warranty)
 443                 return
 444         }
 445         if *version {
 446                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 447                 return
 448         }
 449         if *fsck {
 450                 if !goodIntegrity() {
 451                         os.Exit(1)
 452                 }
 453                 return
 454         }
 455         if *passwdCheck {
 456                 refreshPasswd()
 457                 return
 458         }
 459         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 460                 log.Fatalln("Both -tls-cert and -tls-key are required")
 461         }
 462         refreshPasswd()
 463         log.Println("root:", *root, "bind:", *bind)
 464
 465         ln, err := net.Listen("tcp", *bind)
 466         if err != nil {
 467                 log.Fatal(err)
 468         }
 469         ln = netutil.LimitListener(ln, *maxClients)
 470         server := &http.Server{
 471                 ReadTimeout:  time.Minute,
 472                 WriteTimeout: time.Minute,
 473         }
 474         http.HandleFunc(*norefreshURLPath, handler)
 475         http.HandleFunc(*refreshURLPath, handler)
 476
 477         needsRefreshPasswd := make(chan os.Signal, 0)
 478         needsShutdown := make(chan os.Signal, 0)
 479         exitErr := make(chan error, 0)
 480         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 481         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 482         go func() {
 483                 for range needsRefreshPasswd {
 484                         log.Println("Refreshing passwords")
 485                         refreshPasswd()
 486                 }
 487         }()
 488         go func(s *http.Server) {
 489                 <-needsShutdown
 490                 killed = true
 491                 log.Println("Shutting down")
 492                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 493                 exitErr <- s.Shutdown(ctx)
 494                 cancel()
 495         }(server)
 496
 497         if *tlsCert == "" {
 498                 err = server.Serve(ln)
 499         } else {
 500                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 501         }
 502         if err != http.ErrServerClosed {
 503                 log.Fatal(err)
 504         }
 505         if err := <-exitErr; err != nil {
 506                 log.Fatal(err)
 507         }
 508 }