gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, either version 3 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "crypto/sha256"
  25         "encoding/hex"
  26         "flag"
  27         "fmt"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "path/filepath"
  35         "regexp"
  36         "runtime"
  37         "strings"
  38 )
  39
  40 const (
  41         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  42         HTMLEnd      = "</body></html>"
  43         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  44         SHA256Prefix = "sha256="
  45         SHA256Ext    = ".sha256"
  46         InternalFlag = ".internal"
  47
  48         Warranty = `This program is free software: you can redistribute it and/or modify
  49 it under the terms of the GNU General Public License as published by
  50 the Free Software Foundation, either version 3 of the License, or
  51 (at your option) any later version.
  52
  53 This program is distributed in the hope that it will be useful,
  54 but WITHOUT ANY WARRANTY; without even the implied warranty of
  55 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  56 GNU General Public License for more details.
  57
  58 You should have received a copy of the GNU General Public License
  59 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  60 )
  61
  62 var (
  63         root           = flag.String("root", "./packages", "Path to packages directory")
  64         bind           = flag.String("bind", "[::]:8080", "Address to bind to")
  65         simpleURLPath  = flag.String("simple", "/simple/", "/simple/ URL path")
  66         refreshURLPath = flag.String("refresh", "/refresh/", "Auto-refreshing URL path")
  67         pypiURL        = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  68         auth           = flag.String("auth", "spam:foo", "login:password,...")
  69         fsck           = flag.Bool("fsck", false, "Check integrity of all packages")
  70         version        = flag.Bool("version", false, "Print version information")
  71         warranty       = flag.Bool("warranty", false, "Print warranty information")
  72
  73         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  74         Version string = "UNKNOWN"
  75
  76         passwords map[string]string = make(map[string]string)
  77 )
  78
  79 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  80         path := filepath.Join(*root, dir)
  81         if _, err := os.Stat(path); os.IsNotExist(err) {
  82                 if err = os.Mkdir(path, 0700); err != nil {
  83                         http.Error(w, err.Error(), http.StatusInternalServerError)
  84                         return false
  85                 }
  86                 log.Println(r.RemoteAddr, "mkdir", dir)
  87         }
  88         return true
  89 }
  90
  91 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
  92         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
  93                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
  94                 return true
  95         }
  96         log.Println(r.RemoteAddr, "pypi refresh", dir)
  97         resp, err := http.Get(*pypiURL + dir + "/")
  98         if err != nil {
  99                 http.Error(w, err.Error(), http.StatusBadGateway)
 100                 return false
 101         }
 102         defer resp.Body.Close()
 103         body, err := ioutil.ReadAll(resp.Body)
 104         if err != nil {
 105                 http.Error(w, err.Error(), http.StatusBadGateway)
 106                 return false
 107         }
 108         if !mkdirForPkg(w, r, dir) {
 109                 return false
 110         }
 111         var submatches []string
 112         var uri string
 113         var filename string
 114         var path string
 115         var pkgURL *url.URL
 116         var digest []byte
 117         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 118                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 119                 if len(submatches) == 0 {
 120                         continue
 121                 }
 122                 uri = submatches[1]
 123                 filename = submatches[2]
 124                 if pkgURL, err = url.Parse(uri); err != nil {
 125                         http.Error(w, err.Error(), http.StatusInternalServerError)
 126                         return false
 127                 }
 128                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 129                 if err != nil {
 130                         http.Error(w, err.Error(), http.StatusBadGateway)
 131                         return false
 132                 }
 133                 if filename == filenameGet {
 134                         log.Println(r.RemoteAddr, "pypi download", filename)
 135                         path = filepath.Join(*root, dir, filename)
 136                         resp, err = http.Get(uri)
 137                         if err != nil {
 138                                 http.Error(w, err.Error(), http.StatusBadGateway)
 139                                 return false
 140                         }
 141                         defer resp.Body.Close()
 142                         hasher := sha256.New()
 143                         dst, err := ioutil.TempFile(filepath.Join(*root, dir), "")
 144                         if err != nil {
 145                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 146                                 return false
 147                         }
 148                         wr := io.MultiWriter(hasher, dst)
 149                         if _, err = io.Copy(wr, resp.Body); err != nil {
 150                                 os.Remove(dst.Name())
 151                                 dst.Close()
 152                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 153                                 return false
 154                         }
 155                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 156                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 157                                 os.Remove(dst.Name())
 158                                 dst.Close()
 159                                 http.Error(w, err.Error(), http.StatusBadGateway)
 160                                 return false
 161                         }
 162                         if err = dst.Sync(); err != nil {
 163                                 os.Remove(dst.Name())
 164                                 dst.Close()
 165                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 166                                 return false
 167                         }
 168                         dst.Close()
 169                         if err = os.Rename(dst.Name(), path); err != nil {
 170                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 171                                 return false
 172                         }
 173                 }
 174                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 175                 _, err = os.Stat(path)
 176                 if err == nil {
 177                         continue
 178                 } else {
 179                         if !os.IsNotExist(err) {
 180                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 181                                 return false
 182                         }
 183                 }
 184                 log.Println(r.RemoteAddr, "pypi touch", filename)
 185                 if err = ioutil.WriteFile(path, digest, os.FileMode(0600)); err != nil {
 186                         http.Error(w, err.Error(), http.StatusInternalServerError)
 187                         return false
 188                 }
 189         }
 190         return true
 191 }
 192
 193 func listRoot(w http.ResponseWriter, r *http.Request) {
 194         log.Println(r.RemoteAddr, "root")
 195         files, err := ioutil.ReadDir(*root)
 196         if err != nil {
 197                 http.Error(w, err.Error(), http.StatusInternalServerError)
 198                 return
 199         }
 200         w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root")))
 201         for _, file := range files {
 202                 if file.Mode().IsDir() {
 203                         w.Write([]byte(fmt.Sprintf(
 204                                 HTMLElement,
 205                                 *simpleURLPath+file.Name()+"/",
 206                                 file.Name(),
 207                         )))
 208                 }
 209         }
 210         w.Write([]byte(HTMLEnd))
 211 }
 212
 213 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 214         log.Println(r.RemoteAddr, "dir", dir)
 215         dirPath := filepath.Join(*root, dir)
 216         if autorefresh {
 217                 if !refreshDir(w, r, dir, "") {
 218                         return
 219                 }
 220         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 221                 return
 222         }
 223         files, err := ioutil.ReadDir(dirPath)
 224         if err != nil {
 225                 http.Error(w, err.Error(), http.StatusInternalServerError)
 226                 return
 227         }
 228         w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir)))
 229         var data []byte
 230         var filenameClean string
 231         for _, file := range files {
 232                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 233                         continue
 234                 }
 235                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 236                 if err != nil {
 237                         http.Error(w, err.Error(), http.StatusInternalServerError)
 238                         return
 239                 }
 240                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 241                 w.Write([]byte(fmt.Sprintf(
 242                         HTMLElement,
 243                         strings.Join([]string{
 244                                 *simpleURLPath, dir, "/",
 245                                 filenameClean, "#", SHA256Prefix, string(data),
 246                         }, ""),
 247                         filenameClean,
 248                 )))
 249         }
 250         w.Write([]byte(HTMLEnd))
 251 }
 252
 253 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 254         log.Println(r.RemoteAddr, "pkg", filename)
 255         path := filepath.Join(*root, dir, filename)
 256         if _, err := os.Stat(path); os.IsNotExist(err) {
 257                 if !refreshDir(w, r, dir, filename) {
 258                         return
 259                 }
 260         }
 261         http.ServeFile(w, r, path)
 262 }
 263
 264 func serveUpload(w http.ResponseWriter, r *http.Request) {
 265         username, password, ok := r.BasicAuth()
 266         if !ok || passwords[username] != password {
 267                 log.Println(r.RemoteAddr, "unauthenticated", username)
 268                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 269                 return
 270         }
 271         var err error
 272         if err = r.ParseMultipartForm(1 << 20); err != nil {
 273                 http.Error(w, err.Error(), http.StatusBadRequest)
 274                 return
 275         }
 276         for _, file := range r.MultipartForm.File["content"] {
 277                 filename := file.Filename
 278                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 279                 dir := filename[:strings.LastIndex(filename, "-")]
 280                 dirPath := filepath.Join(*root, dir)
 281                 path := filepath.Join(dirPath, filename)
 282                 if _, err = os.Stat(path); err == nil {
 283                         log.Println(r.RemoteAddr, "already exists", filename)
 284                         http.Error(w, "Already exists", http.StatusBadRequest)
 285                         return
 286                 }
 287                 if !mkdirForPkg(w, r, dir) {
 288                         return
 289                 }
 290                 internalPath := filepath.Join(dirPath, InternalFlag)
 291                 var dst *os.File
 292                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 293                         if dst, err = os.Create(internalPath); err != nil {
 294                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 295                                 return
 296                         }
 297                         dst.Close()
 298                 }
 299                 src, err := file.Open()
 300                 defer src.Close()
 301                 if err != nil {
 302                         http.Error(w, err.Error(), http.StatusInternalServerError)
 303                         return
 304                 }
 305                 dst, err = ioutil.TempFile(dirPath, "")
 306                 if err != nil {
 307                         http.Error(w, err.Error(), http.StatusInternalServerError)
 308                         return
 309                 }
 310                 hasher := sha256.New()
 311                 wr := io.MultiWriter(hasher, dst)
 312                 if _, err = io.Copy(wr, src); err != nil {
 313                         os.Remove(dst.Name())
 314                         dst.Close()
 315                         http.Error(w, err.Error(), http.StatusInternalServerError)
 316                         return
 317                 }
 318                 if err = dst.Sync(); err != nil {
 319                         os.Remove(dst.Name())
 320                         dst.Close()
 321                         http.Error(w, err.Error(), http.StatusInternalServerError)
 322                         return
 323                 }
 324                 dst.Close()
 325                 if err = os.Rename(dst.Name(), path); err != nil {
 326                         http.Error(w, err.Error(), http.StatusInternalServerError)
 327                         return
 328                 }
 329                 if err = ioutil.WriteFile(
 330                         path+SHA256Ext,
 331                         hasher.Sum(nil),
 332                         os.FileMode(0600),
 333                 ); err != nil {
 334                         http.Error(w, err.Error(), http.StatusInternalServerError)
 335                         return
 336                 }
 337         }
 338 }
 339
 340 func handler(w http.ResponseWriter, r *http.Request) {
 341         if r.Method == "GET" {
 342                 var path string
 343                 var autorefresh bool
 344                 if strings.HasPrefix(r.URL.Path, *simpleURLPath) {
 345                         path = strings.TrimPrefix(r.URL.Path, *simpleURLPath)
 346                         autorefresh = false
 347                 } else {
 348                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 349                         autorefresh = true
 350                 }
 351                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 352                 if len(parts) > 2 {
 353                         http.Error(w, "invalid path", http.StatusBadRequest)
 354                         return
 355                 }
 356                 if len(parts) == 1 {
 357                         if parts[0] == "" {
 358                                 listRoot(w, r)
 359                         } else {
 360                                 listDir(w, r, parts[0], autorefresh)
 361                         }
 362                 } else {
 363                         servePkg(w, r, parts[0], parts[1])
 364                 }
 365         } else if r.Method == "POST" {
 366                 serveUpload(w, r)
 367         }
 368 }
 369
 370 func goodIntegrity() bool {
 371         dirs, err := ioutil.ReadDir(*root)
 372         if err != nil {
 373                 log.Fatal(err)
 374         }
 375         hasher := sha256.New()
 376         digest := make([]byte, sha256.Size)
 377         isGood := true
 378         var data []byte
 379         var pkgName string
 380         for _, dir := range dirs {
 381                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 382                 if err != nil {
 383                         log.Fatal(err)
 384                 }
 385                 for _, file := range files {
 386                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 387                                 continue
 388                         }
 389                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 390                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 391                         if err != nil {
 392                                 if os.IsNotExist(err) {
 393                                         continue
 394                                 }
 395                                 log.Fatal(err)
 396                         }
 397                         hasher.Write(data)
 398                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 399                         if err != nil {
 400                                 log.Fatal(err)
 401                         }
 402                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 403                                 log.Println(pkgName, "GOOD")
 404                         } else {
 405                                 isGood = false
 406                                 log.Println(pkgName, "BAD")
 407                         }
 408                         hasher.Reset()
 409                 }
 410         }
 411         return isGood
 412 }
 413
 414 func main() {
 415         flag.Parse()
 416         if *warranty {
 417                 fmt.Println(Warranty)
 418                 return
 419         }
 420         if *version {
 421                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 422                 return
 423         }
 424         if *fsck {
 425                 if !goodIntegrity() {
 426                         os.Exit(1)
 427                 }
 428                 return
 429         }
 430         for _, credentials := range strings.Split(*auth, ",") {
 431                 splitted := strings.Split(credentials, ":")
 432                 if len(splitted) != 2 {
 433                         log.Fatal("Wrong auth format")
 434                 }
 435                 passwords[splitted[0]] = splitted[1]
 436         }
 437         log.Println("root:", *root, "bind:", *bind)
 438         http.HandleFunc(*simpleURLPath, handler)
 439         http.HandleFunc(*refreshURLPath, handler)
 440         log.Fatal(http.ListenAndServe(*bind, nil))
 441 }