gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 // Python private package repository and caching proxy
  19 package main
  20
  21 import (
  22         "bytes"
  23         "crypto/sha256"
  24         "encoding/hex"
  25         "errors"
  26         "flag"
  27         "fmt"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "path/filepath"
  35         "regexp"
  36         "runtime"
  37         "strings"
  38 )
  39
  40 const (
  41         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  42         HTMLEnd      = "</body></html>"
  43         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  44         SHA256Prefix = "sha256="
  45         SHA256Ext    = ".sha256"
  46         InternalFlag = ".internal"
  47
  48         Warranty = `This program is free software: you can redistribute it and/or modify
  49 it under the terms of the GNU General Public License as published by
  50 the Free Software Foundation, version 3 of the License.
  51
  52 This program is distributed in the hope that it will be useful,
  53 but WITHOUT ANY WARRANTY; without even the implied warranty of
  54 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  55 GNU General Public License for more details.
  56
  57 You should have received a copy of the GNU General Public License
  58 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  59 )
  60
  61 var (
  62         root             = flag.String("root", "./packages", "Path to packages directory")
  63         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  64         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  65         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  66         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  67         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  68         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  69         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  70         version          = flag.Bool("version", false, "Print version information")
  71         warranty         = flag.Bool("warranty", false, "Print warranty information")
  72
  73         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  74         Version string = "UNKNOWN"
  75
  76         passwords map[string]Auther = make(map[string]Auther)
  77 )
  78
  79 type Auther interface {
  80         Auth(password string) bool
  81 }
  82
  83 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  84         path := filepath.Join(*root, dir)
  85         if _, err := os.Stat(path); os.IsNotExist(err) {
  86                 if err = os.Mkdir(path, 0700); err != nil {
  87                         http.Error(w, err.Error(), http.StatusInternalServerError)
  88                         return false
  89                 }
  90                 log.Println(r.RemoteAddr, "mkdir", dir)
  91         }
  92         return true
  93 }
  94
  95 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
  96         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
  97                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
  98                 return true
  99         }
 100         log.Println(r.RemoteAddr, "pypi refresh", dir)
 101         resp, err := http.Get(*pypiURL + dir + "/")
 102         if err != nil {
 103                 http.Error(w, err.Error(), http.StatusBadGateway)
 104                 return false
 105         }
 106         defer resp.Body.Close()
 107         body, err := ioutil.ReadAll(resp.Body)
 108         if err != nil {
 109                 http.Error(w, err.Error(), http.StatusBadGateway)
 110                 return false
 111         }
 112         if !mkdirForPkg(w, r, dir) {
 113                 return false
 114         }
 115         var submatches []string
 116         var uri string
 117         var filename string
 118         var path string
 119         var pkgURL *url.URL
 120         var digest []byte
 121         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 122                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 123                 if len(submatches) == 0 {
 124                         continue
 125                 }
 126                 uri = submatches[1]
 127                 filename = submatches[2]
 128                 if pkgURL, err = url.Parse(uri); err != nil {
 129                         http.Error(w, err.Error(), http.StatusInternalServerError)
 130                         return false
 131                 }
 132                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 133                 if err != nil {
 134                         http.Error(w, err.Error(), http.StatusBadGateway)
 135                         return false
 136                 }
 137                 if filename == filenameGet {
 138                         log.Println(r.RemoteAddr, "pypi download", filename)
 139                         path = filepath.Join(*root, dir, filename)
 140                         resp, err = http.Get(uri)
 141                         if err != nil {
 142                                 http.Error(w, err.Error(), http.StatusBadGateway)
 143                                 return false
 144                         }
 145                         defer resp.Body.Close()
 146                         hasher := sha256.New()
 147                         dst, err := ioutil.TempFile(filepath.Join(*root, dir), "")
 148                         if err != nil {
 149                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 150                                 return false
 151                         }
 152                         wr := io.MultiWriter(hasher, dst)
 153                         if _, err = io.Copy(wr, resp.Body); err != nil {
 154                                 os.Remove(dst.Name())
 155                                 dst.Close()
 156                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 157                                 return false
 158                         }
 159                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 160                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 161                                 os.Remove(dst.Name())
 162                                 dst.Close()
 163                                 http.Error(w, err.Error(), http.StatusBadGateway)
 164                                 return false
 165                         }
 166                         if err = dst.Sync(); err != nil {
 167                                 os.Remove(dst.Name())
 168                                 dst.Close()
 169                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 170                                 return false
 171                         }
 172                         dst.Close()
 173                         if err = os.Rename(dst.Name(), path); err != nil {
 174                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 175                                 return false
 176                         }
 177                 }
 178                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 179                 _, err = os.Stat(path)
 180                 if err == nil {
 181                         continue
 182                 } else {
 183                         if !os.IsNotExist(err) {
 184                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 185                                 return false
 186                         }
 187                 }
 188                 log.Println(r.RemoteAddr, "pypi touch", filename)
 189                 if err = ioutil.WriteFile(path, digest, os.FileMode(0600)); err != nil {
 190                         http.Error(w, err.Error(), http.StatusInternalServerError)
 191                         return false
 192                 }
 193         }
 194         return true
 195 }
 196
 197 func listRoot(w http.ResponseWriter, r *http.Request) {
 198         log.Println(r.RemoteAddr, "root")
 199         files, err := ioutil.ReadDir(*root)
 200         if err != nil {
 201                 http.Error(w, err.Error(), http.StatusInternalServerError)
 202                 return
 203         }
 204         w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root")))
 205         for _, file := range files {
 206                 if file.Mode().IsDir() {
 207                         w.Write([]byte(fmt.Sprintf(
 208                                 HTMLElement,
 209                                 *refreshURLPath+file.Name()+"/",
 210                                 file.Name(),
 211                         )))
 212                 }
 213         }
 214         w.Write([]byte(HTMLEnd))
 215 }
 216
 217 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 218         log.Println(r.RemoteAddr, "dir", dir)
 219         dirPath := filepath.Join(*root, dir)
 220         if autorefresh {
 221                 if !refreshDir(w, r, dir, "") {
 222                         return
 223                 }
 224         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 225                 return
 226         }
 227         files, err := ioutil.ReadDir(dirPath)
 228         if err != nil {
 229                 http.Error(w, err.Error(), http.StatusInternalServerError)
 230                 return
 231         }
 232         w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir)))
 233         var data []byte
 234         var filenameClean string
 235         for _, file := range files {
 236                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 237                         continue
 238                 }
 239                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 240                 if err != nil {
 241                         http.Error(w, err.Error(), http.StatusInternalServerError)
 242                         return
 243                 }
 244                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 245                 w.Write([]byte(fmt.Sprintf(
 246                         HTMLElement,
 247                         strings.Join([]string{
 248                                 *refreshURLPath, dir, "/",
 249                                 filenameClean, "#", SHA256Prefix, string(data),
 250                         }, ""),
 251                         filenameClean,
 252                 )))
 253         }
 254         w.Write([]byte(HTMLEnd))
 255 }
 256
 257 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 258         log.Println(r.RemoteAddr, "pkg", filename)
 259         path := filepath.Join(*root, dir, filename)
 260         if _, err := os.Stat(path); os.IsNotExist(err) {
 261                 if !refreshDir(w, r, dir, filename) {
 262                         return
 263                 }
 264         }
 265         http.ServeFile(w, r, path)
 266 }
 267
 268 func strToAuther(verifier string) (string, Auther, error) {
 269         st := strings.SplitN(verifier, "$", 3)
 270         if len(st) != 3 || st[0] != "" {
 271                 return "", nil, errors.New("invalid verifier structure")
 272         }
 273         algorithm := st[1]
 274         var auther Auther
 275         var err error
 276         switch algorithm {
 277         case "argon2i":
 278                 auther, err = parseArgon2i(st[2])
 279         case "sha256":
 280                 auther, err = parseSHA256(st[2])
 281         default:
 282                 err = errors.New("unknown hashing algorithm")
 283         }
 284         return algorithm, auther, err
 285 }
 286
 287 func serveUpload(w http.ResponseWriter, r *http.Request) {
 288         username, password, ok := r.BasicAuth()
 289         if !ok {
 290                 log.Println(r.RemoteAddr, "unauthenticated", username)
 291                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 292                 return
 293         }
 294         auther, ok := passwords[username]
 295         if !ok || !auther.Auth(password) {
 296                 log.Println(r.RemoteAddr, "unauthenticated", username)
 297                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 298                 return
 299         }
 300         var err error
 301         if err = r.ParseMultipartForm(1 << 20); err != nil {
 302                 http.Error(w, err.Error(), http.StatusBadRequest)
 303                 return
 304         }
 305         for _, file := range r.MultipartForm.File["content"] {
 306                 filename := file.Filename
 307                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 308                 dir := filename[:strings.LastIndex(filename, "-")]
 309                 dirPath := filepath.Join(*root, dir)
 310                 path := filepath.Join(dirPath, filename)
 311                 if _, err = os.Stat(path); err == nil {
 312                         log.Println(r.RemoteAddr, "already exists", filename)
 313                         http.Error(w, "Already exists", http.StatusBadRequest)
 314                         return
 315                 }
 316                 if !mkdirForPkg(w, r, dir) {
 317                         return
 318                 }
 319                 internalPath := filepath.Join(dirPath, InternalFlag)
 320                 var dst *os.File
 321                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 322                         if dst, err = os.Create(internalPath); err != nil {
 323                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 324                                 return
 325                         }
 326                         dst.Close()
 327                 }
 328                 src, err := file.Open()
 329                 defer src.Close()
 330                 if err != nil {
 331                         http.Error(w, err.Error(), http.StatusInternalServerError)
 332                         return
 333                 }
 334                 dst, err = ioutil.TempFile(dirPath, "")
 335                 if err != nil {
 336                         http.Error(w, err.Error(), http.StatusInternalServerError)
 337                         return
 338                 }
 339                 hasher := sha256.New()
 340                 wr := io.MultiWriter(hasher, dst)
 341                 if _, err = io.Copy(wr, src); err != nil {
 342                         os.Remove(dst.Name())
 343                         dst.Close()
 344                         http.Error(w, err.Error(), http.StatusInternalServerError)
 345                         return
 346                 }
 347                 if err = dst.Sync(); err != nil {
 348                         os.Remove(dst.Name())
 349                         dst.Close()
 350                         http.Error(w, err.Error(), http.StatusInternalServerError)
 351                         return
 352                 }
 353                 dst.Close()
 354                 if err = os.Rename(dst.Name(), path); err != nil {
 355                         http.Error(w, err.Error(), http.StatusInternalServerError)
 356                         return
 357                 }
 358                 if err = ioutil.WriteFile(
 359                         path+SHA256Ext,
 360                         hasher.Sum(nil),
 361                         os.FileMode(0600),
 362                 ); err != nil {
 363                         http.Error(w, err.Error(), http.StatusInternalServerError)
 364                         return
 365                 }
 366         }
 367 }
 368
 369 func handler(w http.ResponseWriter, r *http.Request) {
 370         if r.Method == "GET" {
 371                 var path string
 372                 var autorefresh bool
 373                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 374                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 375                         autorefresh = false
 376                 } else {
 377                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 378                         autorefresh = true
 379                 }
 380                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 381                 if len(parts) > 2 {
 382                         http.Error(w, "invalid path", http.StatusBadRequest)
 383                         return
 384                 }
 385                 if len(parts) == 1 {
 386                         if parts[0] == "" {
 387                                 listRoot(w, r)
 388                         } else {
 389                                 listDir(w, r, parts[0], autorefresh)
 390                         }
 391                 } else {
 392                         servePkg(w, r, parts[0], parts[1])
 393                 }
 394         } else if r.Method == "POST" {
 395                 serveUpload(w, r)
 396         }
 397 }
 398
 399 func goodIntegrity() bool {
 400         dirs, err := ioutil.ReadDir(*root)
 401         if err != nil {
 402                 log.Fatal(err)
 403         }
 404         hasher := sha256.New()
 405         digest := make([]byte, sha256.Size)
 406         isGood := true
 407         var data []byte
 408         var pkgName string
 409         for _, dir := range dirs {
 410                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 411                 if err != nil {
 412                         log.Fatal(err)
 413                 }
 414                 for _, file := range files {
 415                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 416                                 continue
 417                         }
 418                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 419                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 420                         if err != nil {
 421                                 if os.IsNotExist(err) {
 422                                         continue
 423                                 }
 424                                 log.Fatal(err)
 425                         }
 426                         hasher.Write(data)
 427                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 428                         if err != nil {
 429                                 log.Fatal(err)
 430                         }
 431                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 432                                 log.Println(pkgName, "GOOD")
 433                         } else {
 434                                 isGood = false
 435                                 log.Println(pkgName, "BAD")
 436                         }
 437                         hasher.Reset()
 438                 }
 439         }
 440         return isGood
 441 }
 442
 443 func main() {
 444         flag.Parse()
 445         if *warranty {
 446                 fmt.Println(Warranty)
 447                 return
 448         }
 449         if *version {
 450                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 451                 return
 452         }
 453         if *fsck {
 454                 if !goodIntegrity() {
 455                         os.Exit(1)
 456                 }
 457                 return
 458         }
 459         if *passwdCheck {
 460                 refreshPasswd()
 461                 return
 462         }
 463         refreshPasswd()
 464         log.Println("root:", *root, "bind:", *bind)
 465         http.HandleFunc(*norefreshURLPath, handler)
 466         http.HandleFunc(*refreshURLPath, handler)
 467         log.Fatal(http.ListenAndServe(*bind, nil))
 468 }