gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 // Python private package repository and caching proxy
  19 package main
  20
  21 import (
  22         "bytes"
  23         "crypto/sha256"
  24         "encoding/hex"
  25         "flag"
  26         "fmt"
  27         "io"
  28         "io/ioutil"
  29         "log"
  30         "net/http"
  31         "net/url"
  32         "os"
  33         "os/signal"
  34         "path/filepath"
  35         "regexp"
  36         "runtime"
  37         "strings"
  38         "syscall"
  39 )
  40
  41 const (
  42         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  43         HTMLEnd      = "</body></html>"
  44         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  45         SHA256Prefix = "sha256="
  46         SHA256Ext    = ".sha256"
  47         InternalFlag = ".internal"
  48
  49         Warranty = `This program is free software: you can redistribute it and/or modify
  50 it under the terms of the GNU General Public License as published by
  51 the Free Software Foundation, version 3 of the License.
  52
  53 This program is distributed in the hope that it will be useful,
  54 but WITHOUT ANY WARRANTY; without even the implied warranty of
  55 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  56 GNU General Public License for more details.
  57
  58 You should have received a copy of the GNU General Public License
  59 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  60 )
  61
  62 var (
  63         root             = flag.String("root", "./packages", "Path to packages directory")
  64         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  65         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  66         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  67         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  68         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  69         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  70         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  71         version          = flag.Bool("version", false, "Print version information")
  72         warranty         = flag.Bool("warranty", false, "Print warranty information")
  73
  74         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  75         Version string = "UNKNOWN"
  76
  77         passwords map[string]Auther = make(map[string]Auther)
  78 )
  79
  80 type Auther interface {
  81         Auth(password string) bool
  82 }
  83
  84 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  85         path := filepath.Join(*root, dir)
  86         if _, err := os.Stat(path); os.IsNotExist(err) {
  87                 if err = os.Mkdir(path, 0700); err != nil {
  88                         http.Error(w, err.Error(), http.StatusInternalServerError)
  89                         return false
  90                 }
  91                 log.Println(r.RemoteAddr, "mkdir", dir)
  92         }
  93         return true
  94 }
  95
  96 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
  97         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
  98                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
  99                 return true
 100         }
 101         log.Println(r.RemoteAddr, "pypi refresh", dir)
 102         resp, err := http.Get(*pypiURL + dir + "/")
 103         if err != nil {
 104                 http.Error(w, err.Error(), http.StatusBadGateway)
 105                 return false
 106         }
 107         defer resp.Body.Close()
 108         body, err := ioutil.ReadAll(resp.Body)
 109         if err != nil {
 110                 http.Error(w, err.Error(), http.StatusBadGateway)
 111                 return false
 112         }
 113         if !mkdirForPkg(w, r, dir) {
 114                 return false
 115         }
 116         var submatches []string
 117         var uri string
 118         var filename string
 119         var path string
 120         var pkgURL *url.URL
 121         var digest []byte
 122         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 123                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 124                 if len(submatches) == 0 {
 125                         continue
 126                 }
 127                 uri = submatches[1]
 128                 filename = submatches[2]
 129                 if pkgURL, err = url.Parse(uri); err != nil {
 130                         http.Error(w, err.Error(), http.StatusInternalServerError)
 131                         return false
 132                 }
 133                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 134                 if err != nil {
 135                         http.Error(w, err.Error(), http.StatusBadGateway)
 136                         return false
 137                 }
 138                 if filename == filenameGet {
 139                         log.Println(r.RemoteAddr, "pypi download", filename)
 140                         path = filepath.Join(*root, dir, filename)
 141                         resp, err = http.Get(uri)
 142                         if err != nil {
 143                                 http.Error(w, err.Error(), http.StatusBadGateway)
 144                                 return false
 145                         }
 146                         defer resp.Body.Close()
 147                         hasher := sha256.New()
 148                         dst, err := ioutil.TempFile(filepath.Join(*root, dir), "")
 149                         if err != nil {
 150                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 151                                 return false
 152                         }
 153                         wr := io.MultiWriter(hasher, dst)
 154                         if _, err = io.Copy(wr, resp.Body); err != nil {
 155                                 os.Remove(dst.Name())
 156                                 dst.Close()
 157                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 158                                 return false
 159                         }
 160                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 161                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 162                                 os.Remove(dst.Name())
 163                                 dst.Close()
 164                                 http.Error(w, err.Error(), http.StatusBadGateway)
 165                                 return false
 166                         }
 167                         if err = dst.Sync(); err != nil {
 168                                 os.Remove(dst.Name())
 169                                 dst.Close()
 170                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 171                                 return false
 172                         }
 173                         dst.Close()
 174                         if err = os.Rename(dst.Name(), path); err != nil {
 175                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 176                                 return false
 177                         }
 178                 }
 179                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 180                 _, err = os.Stat(path)
 181                 if err == nil {
 182                         continue
 183                 } else {
 184                         if !os.IsNotExist(err) {
 185                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 186                                 return false
 187                         }
 188                 }
 189                 log.Println(r.RemoteAddr, "pypi touch", filename)
 190                 if err = ioutil.WriteFile(path, digest, os.FileMode(0600)); err != nil {
 191                         http.Error(w, err.Error(), http.StatusInternalServerError)
 192                         return false
 193                 }
 194         }
 195         return true
 196 }
 197
 198 func listRoot(w http.ResponseWriter, r *http.Request) {
 199         log.Println(r.RemoteAddr, "root")
 200         files, err := ioutil.ReadDir(*root)
 201         if err != nil {
 202                 http.Error(w, err.Error(), http.StatusInternalServerError)
 203                 return
 204         }
 205         w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root")))
 206         for _, file := range files {
 207                 if file.Mode().IsDir() {
 208                         w.Write([]byte(fmt.Sprintf(
 209                                 HTMLElement,
 210                                 *refreshURLPath+file.Name()+"/",
 211                                 file.Name(),
 212                         )))
 213                 }
 214         }
 215         w.Write([]byte(HTMLEnd))
 216 }
 217
 218 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 219         log.Println(r.RemoteAddr, "dir", dir)
 220         dirPath := filepath.Join(*root, dir)
 221         if autorefresh {
 222                 if !refreshDir(w, r, dir, "") {
 223                         return
 224                 }
 225         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 226                 return
 227         }
 228         files, err := ioutil.ReadDir(dirPath)
 229         if err != nil {
 230                 http.Error(w, err.Error(), http.StatusInternalServerError)
 231                 return
 232         }
 233         w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir)))
 234         var data []byte
 235         var filenameClean string
 236         for _, file := range files {
 237                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 238                         continue
 239                 }
 240                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 241                 if err != nil {
 242                         http.Error(w, err.Error(), http.StatusInternalServerError)
 243                         return
 244                 }
 245                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 246                 w.Write([]byte(fmt.Sprintf(
 247                         HTMLElement,
 248                         strings.Join([]string{
 249                                 *refreshURLPath, dir, "/",
 250                                 filenameClean, "#", SHA256Prefix, string(data),
 251                         }, ""),
 252                         filenameClean,
 253                 )))
 254         }
 255         w.Write([]byte(HTMLEnd))
 256 }
 257
 258 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 259         log.Println(r.RemoteAddr, "pkg", filename)
 260         path := filepath.Join(*root, dir, filename)
 261         if _, err := os.Stat(path); os.IsNotExist(err) {
 262                 if !refreshDir(w, r, dir, filename) {
 263                         return
 264                 }
 265         }
 266         http.ServeFile(w, r, path)
 267 }
 268
 269 func serveUpload(w http.ResponseWriter, r *http.Request) {
 270         username, password, ok := r.BasicAuth()
 271         if !ok {
 272                 log.Println(r.RemoteAddr, "unauthenticated", username)
 273                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 274                 return
 275         }
 276         auther, ok := passwords[username]
 277         if !ok || !auther.Auth(password) {
 278                 log.Println(r.RemoteAddr, "unauthenticated", username)
 279                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 280                 return
 281         }
 282         var err error
 283         if err = r.ParseMultipartForm(1 << 20); err != nil {
 284                 http.Error(w, err.Error(), http.StatusBadRequest)
 285                 return
 286         }
 287         for _, file := range r.MultipartForm.File["content"] {
 288                 filename := file.Filename
 289                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 290                 dir := filename[:strings.LastIndex(filename, "-")]
 291                 dirPath := filepath.Join(*root, dir)
 292                 path := filepath.Join(dirPath, filename)
 293                 if _, err = os.Stat(path); err == nil {
 294                         log.Println(r.RemoteAddr, "already exists", filename)
 295                         http.Error(w, "Already exists", http.StatusBadRequest)
 296                         return
 297                 }
 298                 if !mkdirForPkg(w, r, dir) {
 299                         return
 300                 }
 301                 internalPath := filepath.Join(dirPath, InternalFlag)
 302                 var dst *os.File
 303                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 304                         if dst, err = os.Create(internalPath); err != nil {
 305                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 306                                 return
 307                         }
 308                         dst.Close()
 309                 }
 310                 src, err := file.Open()
 311                 defer src.Close()
 312                 if err != nil {
 313                         http.Error(w, err.Error(), http.StatusInternalServerError)
 314                         return
 315                 }
 316                 dst, err = ioutil.TempFile(dirPath, "")
 317                 if err != nil {
 318                         http.Error(w, err.Error(), http.StatusInternalServerError)
 319                         return
 320                 }
 321                 hasher := sha256.New()
 322                 wr := io.MultiWriter(hasher, dst)
 323                 if _, err = io.Copy(wr, src); err != nil {
 324                         os.Remove(dst.Name())
 325                         dst.Close()
 326                         http.Error(w, err.Error(), http.StatusInternalServerError)
 327                         return
 328                 }
 329                 if err = dst.Sync(); err != nil {
 330                         os.Remove(dst.Name())
 331                         dst.Close()
 332                         http.Error(w, err.Error(), http.StatusInternalServerError)
 333                         return
 334                 }
 335                 dst.Close()
 336                 if err = os.Rename(dst.Name(), path); err != nil {
 337                         http.Error(w, err.Error(), http.StatusInternalServerError)
 338                         return
 339                 }
 340                 if err = ioutil.WriteFile(
 341                         path+SHA256Ext,
 342                         hasher.Sum(nil),
 343                         os.FileMode(0600),
 344                 ); err != nil {
 345                         http.Error(w, err.Error(), http.StatusInternalServerError)
 346                         return
 347                 }
 348         }
 349 }
 350
 351 func handler(w http.ResponseWriter, r *http.Request) {
 352         if r.Method == "GET" {
 353                 var path string
 354                 var autorefresh bool
 355                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 356                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 357                         autorefresh = false
 358                 } else {
 359                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 360                         autorefresh = true
 361                 }
 362                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 363                 if len(parts) > 2 {
 364                         http.Error(w, "invalid path", http.StatusBadRequest)
 365                         return
 366                 }
 367                 if len(parts) == 1 {
 368                         if parts[0] == "" {
 369                                 listRoot(w, r)
 370                         } else {
 371                                 listDir(w, r, parts[0], autorefresh)
 372                         }
 373                 } else {
 374                         servePkg(w, r, parts[0], parts[1])
 375                 }
 376         } else if r.Method == "POST" {
 377                 serveUpload(w, r)
 378         }
 379 }
 380
 381 func goodIntegrity() bool {
 382         dirs, err := ioutil.ReadDir(*root)
 383         if err != nil {
 384                 log.Fatal(err)
 385         }
 386         hasher := sha256.New()
 387         digest := make([]byte, sha256.Size)
 388         isGood := true
 389         var data []byte
 390         var pkgName string
 391         for _, dir := range dirs {
 392                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 393                 if err != nil {
 394                         log.Fatal(err)
 395                 }
 396                 for _, file := range files {
 397                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 398                                 continue
 399                         }
 400                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 401                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 402                         if err != nil {
 403                                 if os.IsNotExist(err) {
 404                                         continue
 405                                 }
 406                                 log.Fatal(err)
 407                         }
 408                         hasher.Write(data)
 409                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 410                         if err != nil {
 411                                 log.Fatal(err)
 412                         }
 413                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 414                                 log.Println(pkgName, "GOOD")
 415                         } else {
 416                                 isGood = false
 417                                 log.Println(pkgName, "BAD")
 418                         }
 419                         hasher.Reset()
 420                 }
 421         }
 422         return isGood
 423 }
 424
 425 func main() {
 426         flag.Parse()
 427         if *warranty {
 428                 fmt.Println(Warranty)
 429                 return
 430         }
 431         if *version {
 432                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 433                 return
 434         }
 435         if *fsck {
 436                 if !goodIntegrity() {
 437                         os.Exit(1)
 438                 }
 439                 return
 440         }
 441         if *passwdCheck {
 442                 refreshPasswd()
 443                 return
 444         }
 445         refreshPasswd()
 446         log.Println("root:", *root, "bind:", *bind)
 447         needsRefreshPasswd := make(chan os.Signal, 0)
 448         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 449         go func() {
 450                 for range needsRefreshPasswd {
 451                         log.Println("Refreshing passwords")
 452                         refreshPasswd()
 453                 }
 454         }()
 455         http.HandleFunc(*norefreshURLPath, handler)
 456         http.HandleFunc(*refreshURLPath, handler)
 457         log.Fatal(http.ListenAndServe(*bind, nil))
 458 }