gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 // Python private package repository and caching proxy
  19 package main
  20
  21 import (
  22         "bytes"
  23         "crypto/sha256"
  24         "encoding/hex"
  25         "errors"
  26         "flag"
  27         "fmt"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "path/filepath"
  35         "regexp"
  36         "runtime"
  37         "strings"
  38 )
  39
  40 const (
  41         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  42         HTMLEnd      = "</body></html>"
  43         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  44         SHA256Prefix = "sha256="
  45         SHA256Ext    = ".sha256"
  46         InternalFlag = ".internal"
  47
  48         Warranty = `This program is free software: you can redistribute it and/or modify
  49 it under the terms of the GNU General Public License as published by
  50 the Free Software Foundation, version 3 of the License.
  51
  52 This program is distributed in the hope that it will be useful,
  53 but WITHOUT ANY WARRANTY; without even the implied warranty of
  54 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  55 GNU General Public License for more details.
  56
  57 You should have received a copy of the GNU General Public License
  58 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  59 )
  60
  61 var (
  62         root             = flag.String("root", "./packages", "Path to packages directory")
  63         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  64         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  65         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  66         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  67         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  68         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  69         version          = flag.Bool("version", false, "Print version information")
  70         warranty         = flag.Bool("warranty", false, "Print warranty information")
  71
  72         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  73         Version string = "UNKNOWN"
  74
  75         passwords map[string]Auther = make(map[string]Auther)
  76 )
  77
  78 type Auther interface {
  79         Auth(password string) bool
  80 }
  81
  82 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  83         path := filepath.Join(*root, dir)
  84         if _, err := os.Stat(path); os.IsNotExist(err) {
  85                 if err = os.Mkdir(path, 0700); err != nil {
  86                         http.Error(w, err.Error(), http.StatusInternalServerError)
  87                         return false
  88                 }
  89                 log.Println(r.RemoteAddr, "mkdir", dir)
  90         }
  91         return true
  92 }
  93
  94 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
  95         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
  96                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
  97                 return true
  98         }
  99         log.Println(r.RemoteAddr, "pypi refresh", dir)
 100         resp, err := http.Get(*pypiURL + dir + "/")
 101         if err != nil {
 102                 http.Error(w, err.Error(), http.StatusBadGateway)
 103                 return false
 104         }
 105         defer resp.Body.Close()
 106         body, err := ioutil.ReadAll(resp.Body)
 107         if err != nil {
 108                 http.Error(w, err.Error(), http.StatusBadGateway)
 109                 return false
 110         }
 111         if !mkdirForPkg(w, r, dir) {
 112                 return false
 113         }
 114         var submatches []string
 115         var uri string
 116         var filename string
 117         var path string
 118         var pkgURL *url.URL
 119         var digest []byte
 120         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 121                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 122                 if len(submatches) == 0 {
 123                         continue
 124                 }
 125                 uri = submatches[1]
 126                 filename = submatches[2]
 127                 if pkgURL, err = url.Parse(uri); err != nil {
 128                         http.Error(w, err.Error(), http.StatusInternalServerError)
 129                         return false
 130                 }
 131                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 132                 if err != nil {
 133                         http.Error(w, err.Error(), http.StatusBadGateway)
 134                         return false
 135                 }
 136                 if filename == filenameGet {
 137                         log.Println(r.RemoteAddr, "pypi download", filename)
 138                         path = filepath.Join(*root, dir, filename)
 139                         resp, err = http.Get(uri)
 140                         if err != nil {
 141                                 http.Error(w, err.Error(), http.StatusBadGateway)
 142                                 return false
 143                         }
 144                         defer resp.Body.Close()
 145                         hasher := sha256.New()
 146                         dst, err := ioutil.TempFile(filepath.Join(*root, dir), "")
 147                         if err != nil {
 148                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 149                                 return false
 150                         }
 151                         wr := io.MultiWriter(hasher, dst)
 152                         if _, err = io.Copy(wr, resp.Body); err != nil {
 153                                 os.Remove(dst.Name())
 154                                 dst.Close()
 155                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 156                                 return false
 157                         }
 158                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 159                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 160                                 os.Remove(dst.Name())
 161                                 dst.Close()
 162                                 http.Error(w, err.Error(), http.StatusBadGateway)
 163                                 return false
 164                         }
 165                         if err = dst.Sync(); err != nil {
 166                                 os.Remove(dst.Name())
 167                                 dst.Close()
 168                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 169                                 return false
 170                         }
 171                         dst.Close()
 172                         if err = os.Rename(dst.Name(), path); err != nil {
 173                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 174                                 return false
 175                         }
 176                 }
 177                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 178                 _, err = os.Stat(path)
 179                 if err == nil {
 180                         continue
 181                 } else {
 182                         if !os.IsNotExist(err) {
 183                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 184                                 return false
 185                         }
 186                 }
 187                 log.Println(r.RemoteAddr, "pypi touch", filename)
 188                 if err = ioutil.WriteFile(path, digest, os.FileMode(0600)); err != nil {
 189                         http.Error(w, err.Error(), http.StatusInternalServerError)
 190                         return false
 191                 }
 192         }
 193         return true
 194 }
 195
 196 func listRoot(w http.ResponseWriter, r *http.Request) {
 197         log.Println(r.RemoteAddr, "root")
 198         files, err := ioutil.ReadDir(*root)
 199         if err != nil {
 200                 http.Error(w, err.Error(), http.StatusInternalServerError)
 201                 return
 202         }
 203         w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root")))
 204         for _, file := range files {
 205                 if file.Mode().IsDir() {
 206                         w.Write([]byte(fmt.Sprintf(
 207                                 HTMLElement,
 208                                 *refreshURLPath+file.Name()+"/",
 209                                 file.Name(),
 210                         )))
 211                 }
 212         }
 213         w.Write([]byte(HTMLEnd))
 214 }
 215
 216 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 217         log.Println(r.RemoteAddr, "dir", dir)
 218         dirPath := filepath.Join(*root, dir)
 219         if autorefresh {
 220                 if !refreshDir(w, r, dir, "") {
 221                         return
 222                 }
 223         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 224                 return
 225         }
 226         files, err := ioutil.ReadDir(dirPath)
 227         if err != nil {
 228                 http.Error(w, err.Error(), http.StatusInternalServerError)
 229                 return
 230         }
 231         w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir)))
 232         var data []byte
 233         var filenameClean string
 234         for _, file := range files {
 235                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 236                         continue
 237                 }
 238                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 239                 if err != nil {
 240                         http.Error(w, err.Error(), http.StatusInternalServerError)
 241                         return
 242                 }
 243                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 244                 w.Write([]byte(fmt.Sprintf(
 245                         HTMLElement,
 246                         strings.Join([]string{
 247                                 *refreshURLPath, dir, "/",
 248                                 filenameClean, "#", SHA256Prefix, string(data),
 249                         }, ""),
 250                         filenameClean,
 251                 )))
 252         }
 253         w.Write([]byte(HTMLEnd))
 254 }
 255
 256 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 257         log.Println(r.RemoteAddr, "pkg", filename)
 258         path := filepath.Join(*root, dir, filename)
 259         if _, err := os.Stat(path); os.IsNotExist(err) {
 260                 if !refreshDir(w, r, dir, filename) {
 261                         return
 262                 }
 263         }
 264         http.ServeFile(w, r, path)
 265 }
 266
 267 func strToAuther(verifier string) (string, Auther, error) {
 268         st := strings.SplitN(verifier, "$", 3)
 269         if len(st) != 3 || st[0] != "" {
 270                 return "", nil, errors.New("invalid verifier structure")
 271         }
 272         algorithm := st[1]
 273         var auther Auther
 274         var err error
 275         switch algorithm {
 276         case "argon2i":
 277                 auther, err = parseArgon2i(st[2])
 278         case "sha256":
 279                 auther, err = parseSHA256(st[2])
 280         default:
 281                 err = errors.New("unknown hashing algorithm")
 282         }
 283         return algorithm, auther, err
 284 }
 285
 286 func serveUpload(w http.ResponseWriter, r *http.Request) {
 287         username, password, ok := r.BasicAuth()
 288         if !ok {
 289                 log.Println(r.RemoteAddr, "unauthenticated", username)
 290                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 291                 return
 292         }
 293         auther, ok := passwords[username]
 294         if !ok || !auther.Auth(password) {
 295                 log.Println(r.RemoteAddr, "unauthenticated", username)
 296                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 297                 return
 298         }
 299         var err error
 300         if err = r.ParseMultipartForm(1 << 20); err != nil {
 301                 http.Error(w, err.Error(), http.StatusBadRequest)
 302                 return
 303         }
 304         for _, file := range r.MultipartForm.File["content"] {
 305                 filename := file.Filename
 306                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 307                 dir := filename[:strings.LastIndex(filename, "-")]
 308                 dirPath := filepath.Join(*root, dir)
 309                 path := filepath.Join(dirPath, filename)
 310                 if _, err = os.Stat(path); err == nil {
 311                         log.Println(r.RemoteAddr, "already exists", filename)
 312                         http.Error(w, "Already exists", http.StatusBadRequest)
 313                         return
 314                 }
 315                 if !mkdirForPkg(w, r, dir) {
 316                         return
 317                 }
 318                 internalPath := filepath.Join(dirPath, InternalFlag)
 319                 var dst *os.File
 320                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 321                         if dst, err = os.Create(internalPath); err != nil {
 322                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 323                                 return
 324                         }
 325                         dst.Close()
 326                 }
 327                 src, err := file.Open()
 328                 defer src.Close()
 329                 if err != nil {
 330                         http.Error(w, err.Error(), http.StatusInternalServerError)
 331                         return
 332                 }
 333                 dst, err = ioutil.TempFile(dirPath, "")
 334                 if err != nil {
 335                         http.Error(w, err.Error(), http.StatusInternalServerError)
 336                         return
 337                 }
 338                 hasher := sha256.New()
 339                 wr := io.MultiWriter(hasher, dst)
 340                 if _, err = io.Copy(wr, src); err != nil {
 341                         os.Remove(dst.Name())
 342                         dst.Close()
 343                         http.Error(w, err.Error(), http.StatusInternalServerError)
 344                         return
 345                 }
 346                 if err = dst.Sync(); err != nil {
 347                         os.Remove(dst.Name())
 348                         dst.Close()
 349                         http.Error(w, err.Error(), http.StatusInternalServerError)
 350                         return
 351                 }
 352                 dst.Close()
 353                 if err = os.Rename(dst.Name(), path); err != nil {
 354                         http.Error(w, err.Error(), http.StatusInternalServerError)
 355                         return
 356                 }
 357                 if err = ioutil.WriteFile(
 358                         path+SHA256Ext,
 359                         hasher.Sum(nil),
 360                         os.FileMode(0600),
 361                 ); err != nil {
 362                         http.Error(w, err.Error(), http.StatusInternalServerError)
 363                         return
 364                 }
 365         }
 366 }
 367
 368 func handler(w http.ResponseWriter, r *http.Request) {
 369         if r.Method == "GET" {
 370                 var path string
 371                 var autorefresh bool
 372                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 373                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 374                         autorefresh = false
 375                 } else {
 376                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 377                         autorefresh = true
 378                 }
 379                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 380                 if len(parts) > 2 {
 381                         http.Error(w, "invalid path", http.StatusBadRequest)
 382                         return
 383                 }
 384                 if len(parts) == 1 {
 385                         if parts[0] == "" {
 386                                 listRoot(w, r)
 387                         } else {
 388                                 listDir(w, r, parts[0], autorefresh)
 389                         }
 390                 } else {
 391                         servePkg(w, r, parts[0], parts[1])
 392                 }
 393         } else if r.Method == "POST" {
 394                 serveUpload(w, r)
 395         }
 396 }
 397
 398 func goodIntegrity() bool {
 399         dirs, err := ioutil.ReadDir(*root)
 400         if err != nil {
 401                 log.Fatal(err)
 402         }
 403         hasher := sha256.New()
 404         digest := make([]byte, sha256.Size)
 405         isGood := true
 406         var data []byte
 407         var pkgName string
 408         for _, dir := range dirs {
 409                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 410                 if err != nil {
 411                         log.Fatal(err)
 412                 }
 413                 for _, file := range files {
 414                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 415                                 continue
 416                         }
 417                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 418                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 419                         if err != nil {
 420                                 if os.IsNotExist(err) {
 421                                         continue
 422                                 }
 423                                 log.Fatal(err)
 424                         }
 425                         hasher.Write(data)
 426                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 427                         if err != nil {
 428                                 log.Fatal(err)
 429                         }
 430                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 431                                 log.Println(pkgName, "GOOD")
 432                         } else {
 433                                 isGood = false
 434                                 log.Println(pkgName, "BAD")
 435                         }
 436                         hasher.Reset()
 437                 }
 438         }
 439         return isGood
 440 }
 441
 442 func main() {
 443         flag.Parse()
 444         if *warranty {
 445                 fmt.Println(Warranty)
 446                 return
 447         }
 448         if *version {
 449                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 450                 return
 451         }
 452         if *fsck {
 453                 if !goodIntegrity() {
 454                         os.Exit(1)
 455                 }
 456                 return
 457         }
 458         passwd, err := ioutil.ReadFile(*passwdPath)
 459         if err != nil {
 460                 log.Fatal(err)
 461         }
 462         for _, credentials := range strings.Split(strings.TrimRight(string(passwd), "\n"), "\n") {
 463                 splitted := strings.Split(credentials, ":")
 464                 if len(splitted) != 2 {
 465                         log.Fatal("Wrong login:password format")
 466                 }
 467                 _, auther, err := strToAuther(splitted[1])
 468                 if err != nil {
 469                         log.Fatal(err)
 470                 }
 471                 passwords[splitted[0]] = auther
 472                 log.Println("Added password for " + splitted[0])
 473         }
 474         log.Println("root:", *root, "bind:", *bind)
 475         http.HandleFunc(*norefreshURLPath, handler)
 476         http.HandleFunc(*refreshURLPath, handler)
 477         log.Fatal(http.ListenAndServe(*bind, nil))
 478 }