gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 // Python private package repository and caching proxy
  19 package main
  20
  21 import (
  22         "bytes"
  23         "crypto/sha256"
  24         "encoding/hex"
  25         "errors"
  26         "flag"
  27         "fmt"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "os/signal"
  35         "path/filepath"
  36         "regexp"
  37         "runtime"
  38         "strings"
  39         "syscall"
  40 )
  41
  42 const (
  43         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  44         HTMLEnd      = "</body></html>"
  45         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  46         SHA256Prefix = "sha256="
  47         SHA256Ext    = ".sha256"
  48         InternalFlag = ".internal"
  49
  50         Warranty = `This program is free software: you can redistribute it and/or modify
  51 it under the terms of the GNU General Public License as published by
  52 the Free Software Foundation, version 3 of the License.
  53
  54 This program is distributed in the hope that it will be useful,
  55 but WITHOUT ANY WARRANTY; without even the implied warranty of
  56 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  57 GNU General Public License for more details.
  58
  59 You should have received a copy of the GNU General Public License
  60 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  61 )
  62
  63 var (
  64         root             = flag.String("root", "./packages", "Path to packages directory")
  65         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  66         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  67         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  68         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  69         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  70         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  71         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  72         version          = flag.Bool("version", false, "Print version information")
  73         warranty         = flag.Bool("warranty", false, "Print warranty information")
  74
  75         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  76         Version string = "UNKNOWN"
  77
  78         passwords map[string]Auther = make(map[string]Auther)
  79 )
  80
  81 type Auther interface {
  82         Auth(password string) bool
  83 }
  84
  85 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  86         path := filepath.Join(*root, dir)
  87         if _, err := os.Stat(path); os.IsNotExist(err) {
  88                 if err = os.Mkdir(path, 0700); err != nil {
  89                         http.Error(w, err.Error(), http.StatusInternalServerError)
  90                         return false
  91                 }
  92                 log.Println(r.RemoteAddr, "mkdir", dir)
  93         }
  94         return true
  95 }
  96
  97 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
  98         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
  99                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
 100                 return true
 101         }
 102         log.Println(r.RemoteAddr, "pypi refresh", dir)
 103         resp, err := http.Get(*pypiURL + dir + "/")
 104         if err != nil {
 105                 http.Error(w, err.Error(), http.StatusBadGateway)
 106                 return false
 107         }
 108         defer resp.Body.Close()
 109         body, err := ioutil.ReadAll(resp.Body)
 110         if err != nil {
 111                 http.Error(w, err.Error(), http.StatusBadGateway)
 112                 return false
 113         }
 114         if !mkdirForPkg(w, r, dir) {
 115                 return false
 116         }
 117         var submatches []string
 118         var uri string
 119         var filename string
 120         var path string
 121         var pkgURL *url.URL
 122         var digest []byte
 123         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 124                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 125                 if len(submatches) == 0 {
 126                         continue
 127                 }
 128                 uri = submatches[1]
 129                 filename = submatches[2]
 130                 if pkgURL, err = url.Parse(uri); err != nil {
 131                         http.Error(w, err.Error(), http.StatusInternalServerError)
 132                         return false
 133                 }
 134                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 135                 if err != nil {
 136                         http.Error(w, err.Error(), http.StatusBadGateway)
 137                         return false
 138                 }
 139                 if filename == filenameGet {
 140                         log.Println(r.RemoteAddr, "pypi download", filename)
 141                         path = filepath.Join(*root, dir, filename)
 142                         resp, err = http.Get(uri)
 143                         if err != nil {
 144                                 http.Error(w, err.Error(), http.StatusBadGateway)
 145                                 return false
 146                         }
 147                         defer resp.Body.Close()
 148                         hasher := sha256.New()
 149                         dst, err := ioutil.TempFile(filepath.Join(*root, dir), "")
 150                         if err != nil {
 151                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 152                                 return false
 153                         }
 154                         wr := io.MultiWriter(hasher, dst)
 155                         if _, err = io.Copy(wr, resp.Body); err != nil {
 156                                 os.Remove(dst.Name())
 157                                 dst.Close()
 158                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 159                                 return false
 160                         }
 161                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 162                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 163                                 os.Remove(dst.Name())
 164                                 dst.Close()
 165                                 http.Error(w, err.Error(), http.StatusBadGateway)
 166                                 return false
 167                         }
 168                         if err = dst.Sync(); err != nil {
 169                                 os.Remove(dst.Name())
 170                                 dst.Close()
 171                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 172                                 return false
 173                         }
 174                         dst.Close()
 175                         if err = os.Rename(dst.Name(), path); err != nil {
 176                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 177                                 return false
 178                         }
 179                 }
 180                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 181                 _, err = os.Stat(path)
 182                 if err == nil {
 183                         continue
 184                 } else {
 185                         if !os.IsNotExist(err) {
 186                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 187                                 return false
 188                         }
 189                 }
 190                 log.Println(r.RemoteAddr, "pypi touch", filename)
 191                 if err = ioutil.WriteFile(path, digest, os.FileMode(0600)); err != nil {
 192                         http.Error(w, err.Error(), http.StatusInternalServerError)
 193                         return false
 194                 }
 195         }
 196         return true
 197 }
 198
 199 func listRoot(w http.ResponseWriter, r *http.Request) {
 200         log.Println(r.RemoteAddr, "root")
 201         files, err := ioutil.ReadDir(*root)
 202         if err != nil {
 203                 http.Error(w, err.Error(), http.StatusInternalServerError)
 204                 return
 205         }
 206         w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root")))
 207         for _, file := range files {
 208                 if file.Mode().IsDir() {
 209                         w.Write([]byte(fmt.Sprintf(
 210                                 HTMLElement,
 211                                 *refreshURLPath+file.Name()+"/",
 212                                 file.Name(),
 213                         )))
 214                 }
 215         }
 216         w.Write([]byte(HTMLEnd))
 217 }
 218
 219 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 220         log.Println(r.RemoteAddr, "dir", dir)
 221         dirPath := filepath.Join(*root, dir)
 222         if autorefresh {
 223                 if !refreshDir(w, r, dir, "") {
 224                         return
 225                 }
 226         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 227                 return
 228         }
 229         files, err := ioutil.ReadDir(dirPath)
 230         if err != nil {
 231                 http.Error(w, err.Error(), http.StatusInternalServerError)
 232                 return
 233         }
 234         w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir)))
 235         var data []byte
 236         var filenameClean string
 237         for _, file := range files {
 238                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 239                         continue
 240                 }
 241                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 242                 if err != nil {
 243                         http.Error(w, err.Error(), http.StatusInternalServerError)
 244                         return
 245                 }
 246                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 247                 w.Write([]byte(fmt.Sprintf(
 248                         HTMLElement,
 249                         strings.Join([]string{
 250                                 *refreshURLPath, dir, "/",
 251                                 filenameClean, "#", SHA256Prefix, string(data),
 252                         }, ""),
 253                         filenameClean,
 254                 )))
 255         }
 256         w.Write([]byte(HTMLEnd))
 257 }
 258
 259 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 260         log.Println(r.RemoteAddr, "pkg", filename)
 261         path := filepath.Join(*root, dir, filename)
 262         if _, err := os.Stat(path); os.IsNotExist(err) {
 263                 if !refreshDir(w, r, dir, filename) {
 264                         return
 265                 }
 266         }
 267         http.ServeFile(w, r, path)
 268 }
 269
 270 func strToAuther(verifier string) (string, Auther, error) {
 271         st := strings.SplitN(verifier, "$", 3)
 272         if len(st) != 3 || st[0] != "" {
 273                 return "", nil, errors.New("invalid verifier structure")
 274         }
 275         algorithm := st[1]
 276         var auther Auther
 277         var err error
 278         switch algorithm {
 279         case "argon2i":
 280                 auther, err = parseArgon2i(st[2])
 281         case "sha256":
 282                 auther, err = parseSHA256(st[2])
 283         default:
 284                 err = errors.New("unknown hashing algorithm")
 285         }
 286         return algorithm, auther, err
 287 }
 288
 289 func serveUpload(w http.ResponseWriter, r *http.Request) {
 290         username, password, ok := r.BasicAuth()
 291         if !ok {
 292                 log.Println(r.RemoteAddr, "unauthenticated", username)
 293                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 294                 return
 295         }
 296         auther, ok := passwords[username]
 297         if !ok || !auther.Auth(password) {
 298                 log.Println(r.RemoteAddr, "unauthenticated", username)
 299                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 300                 return
 301         }
 302         var err error
 303         if err = r.ParseMultipartForm(1 << 20); err != nil {
 304                 http.Error(w, err.Error(), http.StatusBadRequest)
 305                 return
 306         }
 307         for _, file := range r.MultipartForm.File["content"] {
 308                 filename := file.Filename
 309                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 310                 dir := filename[:strings.LastIndex(filename, "-")]
 311                 dirPath := filepath.Join(*root, dir)
 312                 path := filepath.Join(dirPath, filename)
 313                 if _, err = os.Stat(path); err == nil {
 314                         log.Println(r.RemoteAddr, "already exists", filename)
 315                         http.Error(w, "Already exists", http.StatusBadRequest)
 316                         return
 317                 }
 318                 if !mkdirForPkg(w, r, dir) {
 319                         return
 320                 }
 321                 internalPath := filepath.Join(dirPath, InternalFlag)
 322                 var dst *os.File
 323                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 324                         if dst, err = os.Create(internalPath); err != nil {
 325                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 326                                 return
 327                         }
 328                         dst.Close()
 329                 }
 330                 src, err := file.Open()
 331                 defer src.Close()
 332                 if err != nil {
 333                         http.Error(w, err.Error(), http.StatusInternalServerError)
 334                         return
 335                 }
 336                 dst, err = ioutil.TempFile(dirPath, "")
 337                 if err != nil {
 338                         http.Error(w, err.Error(), http.StatusInternalServerError)
 339                         return
 340                 }
 341                 hasher := sha256.New()
 342                 wr := io.MultiWriter(hasher, dst)
 343                 if _, err = io.Copy(wr, src); err != nil {
 344                         os.Remove(dst.Name())
 345                         dst.Close()
 346                         http.Error(w, err.Error(), http.StatusInternalServerError)
 347                         return
 348                 }
 349                 if err = dst.Sync(); err != nil {
 350                         os.Remove(dst.Name())
 351                         dst.Close()
 352                         http.Error(w, err.Error(), http.StatusInternalServerError)
 353                         return
 354                 }
 355                 dst.Close()
 356                 if err = os.Rename(dst.Name(), path); err != nil {
 357                         http.Error(w, err.Error(), http.StatusInternalServerError)
 358                         return
 359                 }
 360                 if err = ioutil.WriteFile(
 361                         path+SHA256Ext,
 362                         hasher.Sum(nil),
 363                         os.FileMode(0600),
 364                 ); err != nil {
 365                         http.Error(w, err.Error(), http.StatusInternalServerError)
 366                         return
 367                 }
 368         }
 369 }
 370
 371 func handler(w http.ResponseWriter, r *http.Request) {
 372         if r.Method == "GET" {
 373                 var path string
 374                 var autorefresh bool
 375                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 376                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 377                         autorefresh = false
 378                 } else {
 379                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 380                         autorefresh = true
 381                 }
 382                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 383                 if len(parts) > 2 {
 384                         http.Error(w, "invalid path", http.StatusBadRequest)
 385                         return
 386                 }
 387                 if len(parts) == 1 {
 388                         if parts[0] == "" {
 389                                 listRoot(w, r)
 390                         } else {
 391                                 listDir(w, r, parts[0], autorefresh)
 392                         }
 393                 } else {
 394                         servePkg(w, r, parts[0], parts[1])
 395                 }
 396         } else if r.Method == "POST" {
 397                 serveUpload(w, r)
 398         }
 399 }
 400
 401 func goodIntegrity() bool {
 402         dirs, err := ioutil.ReadDir(*root)
 403         if err != nil {
 404                 log.Fatal(err)
 405         }
 406         hasher := sha256.New()
 407         digest := make([]byte, sha256.Size)
 408         isGood := true
 409         var data []byte
 410         var pkgName string
 411         for _, dir := range dirs {
 412                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 413                 if err != nil {
 414                         log.Fatal(err)
 415                 }
 416                 for _, file := range files {
 417                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 418                                 continue
 419                         }
 420                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 421                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 422                         if err != nil {
 423                                 if os.IsNotExist(err) {
 424                                         continue
 425                                 }
 426                                 log.Fatal(err)
 427                         }
 428                         hasher.Write(data)
 429                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 430                         if err != nil {
 431                                 log.Fatal(err)
 432                         }
 433                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 434                                 log.Println(pkgName, "GOOD")
 435                         } else {
 436                                 isGood = false
 437                                 log.Println(pkgName, "BAD")
 438                         }
 439                         hasher.Reset()
 440                 }
 441         }
 442         return isGood
 443 }
 444
 445 func main() {
 446         flag.Parse()
 447         if *warranty {
 448                 fmt.Println(Warranty)
 449                 return
 450         }
 451         if *version {
 452                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 453                 return
 454         }
 455         if *fsck {
 456                 if !goodIntegrity() {
 457                         os.Exit(1)
 458                 }
 459                 return
 460         }
 461         if *passwdCheck {
 462                 refreshPasswd()
 463                 return
 464         }
 465         refreshPasswd()
 466         log.Println("root:", *root, "bind:", *bind)
 467         needsRefreshPasswd := make(chan os.Signal, 0)
 468         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 469         go func() {
 470                 for range needsRefreshPasswd {
 471                         refreshPasswd()
 472                 }
 473         }()
 474         http.HandleFunc(*norefreshURLPath, handler)
 475         http.HandleFunc(*refreshURLPath, handler)
 476         log.Fatal(http.ListenAndServe(*bind, nil))
 477 }