gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "encoding/hex"
  27         "flag"
  28         "fmt"
  29         "io"
  30         "io/ioutil"
  31         "log"
  32         "net"
  33         "net/http"
  34         "net/url"
  35         "os"
  36         "os/signal"
  37         "path/filepath"
  38         "regexp"
  39         "runtime"
  40         "strings"
  41         "syscall"
  42         "time"
  43
  44         "golang.org/x/net/netutil"
  45 )
  46
  47 const (
  48         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  49         HTMLEnd      = "</body></html>"
  50         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  51         SHA256Prefix = "sha256="
  52         SHA256Ext    = ".sha256"
  53         InternalFlag = ".internal"
  54
  55         Warranty = `This program is free software: you can redistribute it and/or modify
  56 it under the terms of the GNU General Public License as published by
  57 the Free Software Foundation, version 3 of the License.
  58
  59 This program is distributed in the hope that it will be useful,
  60 but WITHOUT ANY WARRANTY; without even the implied warranty of
  61 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  62 GNU General Public License for more details.
  63
  64 You should have received a copy of the GNU General Public License
  65 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  66 )
  67
  68 var (
  69         root             = flag.String("root", "./packages", "Path to packages directory")
  70         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  71         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  72         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  73         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  74         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  75         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  76         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  77         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  78         version          = flag.Bool("version", false, "Print version information")
  79         warranty         = flag.Bool("warranty", false, "Print warranty information")
  80
  81         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  82         Version string = "UNKNOWN"
  83
  84         passwords map[string]Auther = make(map[string]Auther)
  85 )
  86
  87 type Auther interface {
  88         Auth(password string) bool
  89 }
  90
  91 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  92         path := filepath.Join(*root, dir)
  93         if _, err := os.Stat(path); os.IsNotExist(err) {
  94                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
  95                         http.Error(w, err.Error(), http.StatusInternalServerError)
  96                         return false
  97                 }
  98                 log.Println(r.RemoteAddr, "mkdir", dir)
  99         }
 100         return true
 101 }
 102
 103 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
 104         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
 105                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
 106                 return true
 107         }
 108         log.Println(r.RemoteAddr, "pypi refresh", dir)
 109         resp, err := http.Get(*pypiURL + dir + "/")
 110         if err != nil {
 111                 http.Error(w, err.Error(), http.StatusBadGateway)
 112                 return false
 113         }
 114         defer resp.Body.Close()
 115         body, err := ioutil.ReadAll(resp.Body)
 116         if err != nil {
 117                 http.Error(w, err.Error(), http.StatusBadGateway)
 118                 return false
 119         }
 120         if !mkdirForPkg(w, r, dir) {
 121                 return false
 122         }
 123         var submatches []string
 124         var uri string
 125         var filename string
 126         var path string
 127         var pkgURL *url.URL
 128         var digest []byte
 129         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 130                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 131                 if len(submatches) == 0 {
 132                         continue
 133                 }
 134                 uri = submatches[1]
 135                 filename = submatches[2]
 136                 if pkgURL, err = url.Parse(uri); err != nil {
 137                         http.Error(w, err.Error(), http.StatusInternalServerError)
 138                         return false
 139                 }
 140                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 141                 if err != nil {
 142                         http.Error(w, err.Error(), http.StatusBadGateway)
 143                         return false
 144                 }
 145                 if filename == filenameGet {
 146                         log.Println(r.RemoteAddr, "pypi download", filename)
 147                         path = filepath.Join(*root, dir, filename)
 148                         resp, err = http.Get(uri)
 149                         if err != nil {
 150                                 http.Error(w, err.Error(), http.StatusBadGateway)
 151                                 return false
 152                         }
 153                         defer resp.Body.Close()
 154                         hasher := sha256.New()
 155                         dst, err := ioutil.TempFile(filepath.Join(*root, dir), "")
 156                         if err != nil {
 157                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 158                                 return false
 159                         }
 160                         wr := io.MultiWriter(hasher, dst)
 161                         if _, err = io.Copy(wr, resp.Body); err != nil {
 162                                 os.Remove(dst.Name())
 163                                 dst.Close()
 164                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 165                                 return false
 166                         }
 167                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 168                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 169                                 os.Remove(dst.Name())
 170                                 dst.Close()
 171                                 http.Error(w, err.Error(), http.StatusBadGateway)
 172                                 return false
 173                         }
 174                         if err = dst.Sync(); err != nil {
 175                                 os.Remove(dst.Name())
 176                                 dst.Close()
 177                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 178                                 return false
 179                         }
 180                         dst.Close()
 181                         if err = os.Rename(dst.Name(), path); err != nil {
 182                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 183                                 return false
 184                         }
 185                 }
 186                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 187                 _, err = os.Stat(path)
 188                 if err == nil {
 189                         continue
 190                 } else {
 191                         if !os.IsNotExist(err) {
 192                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 193                                 return false
 194                         }
 195                 }
 196                 log.Println(r.RemoteAddr, "pypi touch", filename)
 197                 if err = ioutil.WriteFile(path, digest, os.FileMode(0666)); err != nil {
 198                         http.Error(w, err.Error(), http.StatusInternalServerError)
 199                         return false
 200                 }
 201         }
 202         return true
 203 }
 204
 205 func listRoot(w http.ResponseWriter, r *http.Request) {
 206         log.Println(r.RemoteAddr, "root")
 207         files, err := ioutil.ReadDir(*root)
 208         if err != nil {
 209                 http.Error(w, err.Error(), http.StatusInternalServerError)
 210                 return
 211         }
 212         w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root")))
 213         for _, file := range files {
 214                 if file.Mode().IsDir() {
 215                         w.Write([]byte(fmt.Sprintf(
 216                                 HTMLElement,
 217                                 *refreshURLPath+file.Name()+"/",
 218                                 file.Name(),
 219                         )))
 220                 }
 221         }
 222         w.Write([]byte(HTMLEnd))
 223 }
 224
 225 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 226         log.Println(r.RemoteAddr, "dir", dir)
 227         dirPath := filepath.Join(*root, dir)
 228         if autorefresh {
 229                 if !refreshDir(w, r, dir, "") {
 230                         return
 231                 }
 232         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 233                 return
 234         }
 235         files, err := ioutil.ReadDir(dirPath)
 236         if err != nil {
 237                 http.Error(w, err.Error(), http.StatusInternalServerError)
 238                 return
 239         }
 240         w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir)))
 241         var data []byte
 242         var filenameClean string
 243         for _, file := range files {
 244                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 245                         continue
 246                 }
 247                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 248                 if err != nil {
 249                         http.Error(w, err.Error(), http.StatusInternalServerError)
 250                         return
 251                 }
 252                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 253                 w.Write([]byte(fmt.Sprintf(
 254                         HTMLElement,
 255                         strings.Join([]string{
 256                                 *refreshURLPath, dir, "/",
 257                                 filenameClean, "#", SHA256Prefix, string(data),
 258                         }, ""),
 259                         filenameClean,
 260                 )))
 261         }
 262         w.Write([]byte(HTMLEnd))
 263 }
 264
 265 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 266         log.Println(r.RemoteAddr, "pkg", filename)
 267         path := filepath.Join(*root, dir, filename)
 268         if _, err := os.Stat(path); os.IsNotExist(err) {
 269                 if !refreshDir(w, r, dir, filename) {
 270                         return
 271                 }
 272         }
 273         http.ServeFile(w, r, path)
 274 }
 275
 276 func serveUpload(w http.ResponseWriter, r *http.Request) {
 277         username, password, ok := r.BasicAuth()
 278         if !ok {
 279                 log.Println(r.RemoteAddr, "unauthenticated", username)
 280                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 281                 return
 282         }
 283         auther, ok := passwords[username]
 284         if !ok || !auther.Auth(password) {
 285                 log.Println(r.RemoteAddr, "unauthenticated", username)
 286                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 287                 return
 288         }
 289         var err error
 290         if err = r.ParseMultipartForm(1 << 20); err != nil {
 291                 http.Error(w, err.Error(), http.StatusBadRequest)
 292                 return
 293         }
 294         for _, file := range r.MultipartForm.File["content"] {
 295                 filename := file.Filename
 296                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 297                 dir := filename[:strings.LastIndex(filename, "-")]
 298                 dirPath := filepath.Join(*root, dir)
 299                 path := filepath.Join(dirPath, filename)
 300                 if _, err = os.Stat(path); err == nil {
 301                         log.Println(r.RemoteAddr, "already exists", filename)
 302                         http.Error(w, "Already exists", http.StatusBadRequest)
 303                         return
 304                 }
 305                 if !mkdirForPkg(w, r, dir) {
 306                         return
 307                 }
 308                 internalPath := filepath.Join(dirPath, InternalFlag)
 309                 var dst *os.File
 310                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 311                         if dst, err = os.Create(internalPath); err != nil {
 312                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 313                                 return
 314                         }
 315                         dst.Close()
 316                 }
 317                 src, err := file.Open()
 318                 defer src.Close()
 319                 if err != nil {
 320                         http.Error(w, err.Error(), http.StatusInternalServerError)
 321                         return
 322                 }
 323                 dst, err = ioutil.TempFile(dirPath, "")
 324                 if err != nil {
 325                         http.Error(w, err.Error(), http.StatusInternalServerError)
 326                         return
 327                 }
 328                 hasher := sha256.New()
 329                 wr := io.MultiWriter(hasher, dst)
 330                 if _, err = io.Copy(wr, src); err != nil {
 331                         os.Remove(dst.Name())
 332                         dst.Close()
 333                         http.Error(w, err.Error(), http.StatusInternalServerError)
 334                         return
 335                 }
 336                 if err = dst.Sync(); err != nil {
 337                         os.Remove(dst.Name())
 338                         dst.Close()
 339                         http.Error(w, err.Error(), http.StatusInternalServerError)
 340                         return
 341                 }
 342                 dst.Close()
 343                 if err = os.Rename(dst.Name(), path); err != nil {
 344                         http.Error(w, err.Error(), http.StatusInternalServerError)
 345                         return
 346                 }
 347                 if err = ioutil.WriteFile(path+SHA256Ext, hasher.Sum(nil), os.FileMode(0666)); err != nil {
 348                         http.Error(w, err.Error(), http.StatusInternalServerError)
 349                         return
 350                 }
 351         }
 352 }
 353
 354 func handler(w http.ResponseWriter, r *http.Request) {
 355         if r.Method == "GET" {
 356                 var path string
 357                 var autorefresh bool
 358                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 359                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 360                         autorefresh = false
 361                 } else {
 362                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 363                         autorefresh = true
 364                 }
 365                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 366                 if len(parts) > 2 {
 367                         http.Error(w, "invalid path", http.StatusBadRequest)
 368                         return
 369                 }
 370                 if len(parts) == 1 {
 371                         if parts[0] == "" {
 372                                 listRoot(w, r)
 373                         } else {
 374                                 listDir(w, r, parts[0], autorefresh)
 375                         }
 376                 } else {
 377                         servePkg(w, r, parts[0], parts[1])
 378                 }
 379         } else if r.Method == "POST" {
 380                 serveUpload(w, r)
 381         }
 382 }
 383
 384 func goodIntegrity() bool {
 385         dirs, err := ioutil.ReadDir(*root)
 386         if err != nil {
 387                 log.Fatal(err)
 388         }
 389         hasher := sha256.New()
 390         digest := make([]byte, sha256.Size)
 391         isGood := true
 392         var data []byte
 393         var pkgName string
 394         for _, dir := range dirs {
 395                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 396                 if err != nil {
 397                         log.Fatal(err)
 398                 }
 399                 for _, file := range files {
 400                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 401                                 continue
 402                         }
 403                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 404                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 405                         if err != nil {
 406                                 if os.IsNotExist(err) {
 407                                         continue
 408                                 }
 409                                 log.Fatal(err)
 410                         }
 411                         hasher.Write(data)
 412                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 413                         if err != nil {
 414                                 log.Fatal(err)
 415                         }
 416                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 417                                 log.Println(pkgName, "GOOD")
 418                         } else {
 419                                 isGood = false
 420                                 log.Println(pkgName, "BAD")
 421                         }
 422                         hasher.Reset()
 423                 }
 424         }
 425         return isGood
 426 }
 427
 428 func main() {
 429         flag.Parse()
 430         if *warranty {
 431                 fmt.Println(Warranty)
 432                 return
 433         }
 434         if *version {
 435                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 436                 return
 437         }
 438         if *fsck {
 439                 if !goodIntegrity() {
 440                         os.Exit(1)
 441                 }
 442                 return
 443         }
 444         if *passwdCheck {
 445                 refreshPasswd()
 446                 return
 447         }
 448         refreshPasswd()
 449         log.Println("root:", *root, "bind:", *bind)
 450         needsRefreshPasswd := make(chan os.Signal, 0)
 451         needsShutdown := make(chan os.Signal, 0)
 452         killed := make(chan error, 0)
 453         http.HandleFunc(*norefreshURLPath, handler)
 454         http.HandleFunc(*refreshURLPath, handler)
 455         ln, err := net.Listen("tcp", *bind)
 456         if err != nil {
 457                 log.Fatal(err)
 458         }
 459         s := &http.Server{
 460                 ReadTimeout:  time.Minute,
 461                 WriteTimeout: time.Minute,
 462         }
 463         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 464         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 465         go func() {
 466                 for range needsRefreshPasswd {
 467                         log.Println("Refreshing passwords")
 468                         refreshPasswd()
 469                 }
 470         }()
 471         go func(s *http.Server) {
 472                 <-needsShutdown
 473                 log.Println("Shutting down")
 474                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 475                 killed <- s.Shutdown(ctx)
 476                 cancel()
 477         }(s)
 478         if err := s.Serve(netutil.LimitListener(ln, *maxClients)); err != http.ErrServerClosed {
 479                 log.Fatal(err)
 480         }
 481         if err := <-killed; err != nil {
 482                 log.Fatal(err)
 483         }
 484 }