gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "encoding/hex"
  27         "flag"
  28         "fmt"
  29         "io"
  30         "io/ioutil"
  31         "log"
  32         "net"
  33         "net/http"
  34         "net/url"
  35         "os"
  36         "os/signal"
  37         "path/filepath"
  38         "regexp"
  39         "runtime"
  40         "strings"
  41         "syscall"
  42         "time"
  43
  44         "golang.org/x/net/netutil"
  45 )
  46
  47 const (
  48         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  49         HTMLEnd      = "</body></html>"
  50         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  51         SHA256Prefix = "sha256="
  52         SHA256Ext    = ".sha256"
  53         InternalFlag = ".internal"
  54
  55         Warranty = `This program is free software: you can redistribute it and/or modify
  56 it under the terms of the GNU General Public License as published by
  57 the Free Software Foundation, version 3 of the License.
  58
  59 This program is distributed in the hope that it will be useful,
  60 but WITHOUT ANY WARRANTY; without even the implied warranty of
  61 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  62 GNU General Public License for more details.
  63
  64 You should have received a copy of the GNU General Public License
  65 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  66 )
  67
  68 var (
  69         root             = flag.String("root", "./packages", "Path to packages directory")
  70         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  71         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
  72         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
  73         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  74         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  75         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  76         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  77         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  78         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  79         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  80         version          = flag.Bool("version", false, "Print version information")
  81         warranty         = flag.Bool("warranty", false, "Print warranty information")
  82
  83         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  84         Version string = "UNKNOWN"
  85
  86         passwords map[string]Auther = make(map[string]Auther)
  87 )
  88
  89 type Auther interface {
  90         Auth(password string) bool
  91 }
  92
  93 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  94         path := filepath.Join(*root, dir)
  95         if _, err := os.Stat(path); os.IsNotExist(err) {
  96                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
  97                         http.Error(w, err.Error(), http.StatusInternalServerError)
  98                         return false
  99                 }
 100                 log.Println(r.RemoteAddr, "mkdir", dir)
 101         }
 102         return true
 103 }
 104
 105 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
 106         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
 107                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
 108                 return true
 109         }
 110         log.Println(r.RemoteAddr, "pypi refresh", dir)
 111         resp, err := http.Get(*pypiURL + dir + "/")
 112         if err != nil {
 113                 http.Error(w, err.Error(), http.StatusBadGateway)
 114                 return false
 115         }
 116         defer resp.Body.Close()
 117         body, err := ioutil.ReadAll(resp.Body)
 118         if err != nil {
 119                 http.Error(w, err.Error(), http.StatusBadGateway)
 120                 return false
 121         }
 122         if !mkdirForPkg(w, r, dir) {
 123                 return false
 124         }
 125         var submatches []string
 126         var uri string
 127         var filename string
 128         var path string
 129         var pkgURL *url.URL
 130         var digest []byte
 131         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 132                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 133                 if len(submatches) == 0 {
 134                         continue
 135                 }
 136                 uri = submatches[1]
 137                 filename = submatches[2]
 138                 if pkgURL, err = url.Parse(uri); err != nil {
 139                         http.Error(w, err.Error(), http.StatusInternalServerError)
 140                         return false
 141                 }
 142                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 143                 if err != nil {
 144                         http.Error(w, err.Error(), http.StatusBadGateway)
 145                         return false
 146                 }
 147                 if filename == filenameGet {
 148                         log.Println(r.RemoteAddr, "pypi download", filename)
 149                         path = filepath.Join(*root, dir, filename)
 150                         resp, err = http.Get(uri)
 151                         if err != nil {
 152                                 http.Error(w, err.Error(), http.StatusBadGateway)
 153                                 return false
 154                         }
 155                         defer resp.Body.Close()
 156                         hasher := sha256.New()
 157                         dst, err := ioutil.TempFile(filepath.Join(*root, dir), "")
 158                         if err != nil {
 159                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 160                                 return false
 161                         }
 162                         wr := io.MultiWriter(hasher, dst)
 163                         if _, err = io.Copy(wr, resp.Body); err != nil {
 164                                 os.Remove(dst.Name())
 165                                 dst.Close()
 166                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 167                                 return false
 168                         }
 169                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 170                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 171                                 os.Remove(dst.Name())
 172                                 dst.Close()
 173                                 http.Error(w, err.Error(), http.StatusBadGateway)
 174                                 return false
 175                         }
 176                         if err = dst.Sync(); err != nil {
 177                                 os.Remove(dst.Name())
 178                                 dst.Close()
 179                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 180                                 return false
 181                         }
 182                         dst.Close()
 183                         if err = os.Rename(dst.Name(), path); err != nil {
 184                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 185                                 return false
 186                         }
 187                 }
 188                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 189                 _, err = os.Stat(path)
 190                 if err == nil {
 191                         continue
 192                 } else {
 193                         if !os.IsNotExist(err) {
 194                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 195                                 return false
 196                         }
 197                 }
 198                 log.Println(r.RemoteAddr, "pypi touch", filename)
 199                 if err = ioutil.WriteFile(path, digest, os.FileMode(0666)); err != nil {
 200                         http.Error(w, err.Error(), http.StatusInternalServerError)
 201                         return false
 202                 }
 203         }
 204         return true
 205 }
 206
 207 func listRoot(w http.ResponseWriter, r *http.Request) {
 208         log.Println(r.RemoteAddr, "root")
 209         files, err := ioutil.ReadDir(*root)
 210         if err != nil {
 211                 http.Error(w, err.Error(), http.StatusInternalServerError)
 212                 return
 213         }
 214         w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root")))
 215         for _, file := range files {
 216                 if file.Mode().IsDir() {
 217                         w.Write([]byte(fmt.Sprintf(
 218                                 HTMLElement,
 219                                 *refreshURLPath+file.Name()+"/",
 220                                 file.Name(),
 221                         )))
 222                 }
 223         }
 224         w.Write([]byte(HTMLEnd))
 225 }
 226
 227 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 228         log.Println(r.RemoteAddr, "dir", dir)
 229         dirPath := filepath.Join(*root, dir)
 230         if autorefresh {
 231                 if !refreshDir(w, r, dir, "") {
 232                         return
 233                 }
 234         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 235                 return
 236         }
 237         files, err := ioutil.ReadDir(dirPath)
 238         if err != nil {
 239                 http.Error(w, err.Error(), http.StatusInternalServerError)
 240                 return
 241         }
 242         w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir)))
 243         var data []byte
 244         var filenameClean string
 245         for _, file := range files {
 246                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 247                         continue
 248                 }
 249                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 250                 if err != nil {
 251                         http.Error(w, err.Error(), http.StatusInternalServerError)
 252                         return
 253                 }
 254                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 255                 w.Write([]byte(fmt.Sprintf(
 256                         HTMLElement,
 257                         strings.Join([]string{
 258                                 *refreshURLPath, dir, "/",
 259                                 filenameClean, "#", SHA256Prefix, string(data),
 260                         }, ""),
 261                         filenameClean,
 262                 )))
 263         }
 264         w.Write([]byte(HTMLEnd))
 265 }
 266
 267 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 268         log.Println(r.RemoteAddr, "pkg", filename)
 269         path := filepath.Join(*root, dir, filename)
 270         if _, err := os.Stat(path); os.IsNotExist(err) {
 271                 if !refreshDir(w, r, dir, filename) {
 272                         return
 273                 }
 274         }
 275         http.ServeFile(w, r, path)
 276 }
 277
 278 func serveUpload(w http.ResponseWriter, r *http.Request) {
 279         username, password, ok := r.BasicAuth()
 280         if !ok {
 281                 log.Println(r.RemoteAddr, "unauthenticated", username)
 282                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 283                 return
 284         }
 285         auther, ok := passwords[username]
 286         if !ok || !auther.Auth(password) {
 287                 log.Println(r.RemoteAddr, "unauthenticated", username)
 288                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 289                 return
 290         }
 291         var err error
 292         if err = r.ParseMultipartForm(1 << 20); err != nil {
 293                 http.Error(w, err.Error(), http.StatusBadRequest)
 294                 return
 295         }
 296         for _, file := range r.MultipartForm.File["content"] {
 297                 filename := file.Filename
 298                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 299                 dir := filename[:strings.LastIndex(filename, "-")]
 300                 dirPath := filepath.Join(*root, dir)
 301                 path := filepath.Join(dirPath, filename)
 302                 if _, err = os.Stat(path); err == nil {
 303                         log.Println(r.RemoteAddr, "already exists", filename)
 304                         http.Error(w, "Already exists", http.StatusBadRequest)
 305                         return
 306                 }
 307                 if !mkdirForPkg(w, r, dir) {
 308                         return
 309                 }
 310                 internalPath := filepath.Join(dirPath, InternalFlag)
 311                 var dst *os.File
 312                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 313                         if dst, err = os.Create(internalPath); err != nil {
 314                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 315                                 return
 316                         }
 317                         dst.Close()
 318                 }
 319                 src, err := file.Open()
 320                 defer src.Close()
 321                 if err != nil {
 322                         http.Error(w, err.Error(), http.StatusInternalServerError)
 323                         return
 324                 }
 325                 dst, err = ioutil.TempFile(dirPath, "")
 326                 if err != nil {
 327                         http.Error(w, err.Error(), http.StatusInternalServerError)
 328                         return
 329                 }
 330                 hasher := sha256.New()
 331                 wr := io.MultiWriter(hasher, dst)
 332                 if _, err = io.Copy(wr, src); err != nil {
 333                         os.Remove(dst.Name())
 334                         dst.Close()
 335                         http.Error(w, err.Error(), http.StatusInternalServerError)
 336                         return
 337                 }
 338                 if err = dst.Sync(); err != nil {
 339                         os.Remove(dst.Name())
 340                         dst.Close()
 341                         http.Error(w, err.Error(), http.StatusInternalServerError)
 342                         return
 343                 }
 344                 dst.Close()
 345                 if err = os.Rename(dst.Name(), path); err != nil {
 346                         http.Error(w, err.Error(), http.StatusInternalServerError)
 347                         return
 348                 }
 349                 if err = ioutil.WriteFile(path+SHA256Ext, hasher.Sum(nil), os.FileMode(0666)); err != nil {
 350                         http.Error(w, err.Error(), http.StatusInternalServerError)
 351                         return
 352                 }
 353         }
 354 }
 355
 356 func handler(w http.ResponseWriter, r *http.Request) {
 357         if r.Method == "GET" {
 358                 var path string
 359                 var autorefresh bool
 360                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 361                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 362                         autorefresh = false
 363                 } else {
 364                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 365                         autorefresh = true
 366                 }
 367                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 368                 if len(parts) > 2 {
 369                         http.Error(w, "invalid path", http.StatusBadRequest)
 370                         return
 371                 }
 372                 if len(parts) == 1 {
 373                         if parts[0] == "" {
 374                                 listRoot(w, r)
 375                         } else {
 376                                 listDir(w, r, parts[0], autorefresh)
 377                         }
 378                 } else {
 379                         servePkg(w, r, parts[0], parts[1])
 380                 }
 381         } else if r.Method == "POST" {
 382                 serveUpload(w, r)
 383         }
 384 }
 385
 386 func goodIntegrity() bool {
 387         dirs, err := ioutil.ReadDir(*root)
 388         if err != nil {
 389                 log.Fatal(err)
 390         }
 391         hasher := sha256.New()
 392         digest := make([]byte, sha256.Size)
 393         isGood := true
 394         var data []byte
 395         var pkgName string
 396         for _, dir := range dirs {
 397                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 398                 if err != nil {
 399                         log.Fatal(err)
 400                 }
 401                 for _, file := range files {
 402                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 403                                 continue
 404                         }
 405                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 406                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 407                         if err != nil {
 408                                 if os.IsNotExist(err) {
 409                                         continue
 410                                 }
 411                                 log.Fatal(err)
 412                         }
 413                         hasher.Write(data)
 414                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 415                         if err != nil {
 416                                 log.Fatal(err)
 417                         }
 418                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 419                                 log.Println(pkgName, "GOOD")
 420                         } else {
 421                                 isGood = false
 422                                 log.Println(pkgName, "BAD")
 423                         }
 424                         hasher.Reset()
 425                 }
 426         }
 427         return isGood
 428 }
 429
 430 func main() {
 431         flag.Parse()
 432         if *warranty {
 433                 fmt.Println(Warranty)
 434                 return
 435         }
 436         if *version {
 437                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 438                 return
 439         }
 440         if *fsck {
 441                 if !goodIntegrity() {
 442                         os.Exit(1)
 443                 }
 444                 return
 445         }
 446         if *passwdCheck {
 447                 refreshPasswd()
 448                 return
 449         }
 450         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 451                 log.Fatalln("Both -tls-cert and -tls-key are required")
 452         }
 453         refreshPasswd()
 454         log.Println("root:", *root, "bind:", *bind)
 455
 456         ln, err := net.Listen("tcp", *bind)
 457         if err != nil {
 458                 log.Fatal(err)
 459         }
 460         ln = netutil.LimitListener(ln, *maxClients)
 461         server := &http.Server{
 462                 ReadTimeout:  time.Minute,
 463                 WriteTimeout: time.Minute,
 464         }
 465         http.HandleFunc(*norefreshURLPath, handler)
 466         http.HandleFunc(*refreshURLPath, handler)
 467
 468         needsRefreshPasswd := make(chan os.Signal, 0)
 469         needsShutdown := make(chan os.Signal, 0)
 470         killed := make(chan error, 0)
 471         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 472         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 473         go func() {
 474                 for range needsRefreshPasswd {
 475                         log.Println("Refreshing passwords")
 476                         refreshPasswd()
 477                 }
 478         }()
 479         go func(s *http.Server) {
 480                 <-needsShutdown
 481                 log.Println("Shutting down")
 482                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 483                 killed <- s.Shutdown(ctx)
 484                 cancel()
 485         }(server)
 486
 487         if *tlsCert == "" {
 488                 err = server.Serve(ln)
 489         } else {
 490                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 491         }
 492         if err != http.ErrServerClosed {
 493                 log.Fatal(err)
 494         }
 495         if err := <-killed; err != nil {
 496                 log.Fatal(err)
 497         }
 498 }