gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 // Python private package repository and caching proxy
  19 package main
  20
  21 import (
  22         "bytes"
  23         "crypto/sha256"
  24         "encoding/hex"
  25         "flag"
  26         "fmt"
  27         "io"
  28         "io/ioutil"
  29         "log"
  30         "net/http"
  31         "net/url"
  32         "os"
  33         "path/filepath"
  34         "regexp"
  35         "runtime"
  36         "strings"
  37 )
  38
  39 const (
  40         HTMLBegin    = "<!DOCTYPE html><html><head><title>Links for %s</title></head><body><h1>Links for %s</h1>\n"
  41         HTMLEnd      = "</body></html>"
  42         HTMLElement  = "<a href='%s'>%s</a><br/>\n"
  43         SHA256Prefix = "sha256="
  44         SHA256Ext    = ".sha256"
  45         InternalFlag = ".internal"
  46
  47         Warranty = `This program is free software: you can redistribute it and/or modify
  48 it under the terms of the GNU General Public License as published by
  49 the Free Software Foundation, version 3 of the License.
  50
  51 This program is distributed in the hope that it will be useful,
  52 but WITHOUT ANY WARRANTY; without even the implied warranty of
  53 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  54 GNU General Public License for more details.
  55
  56 You should have received a copy of the GNU General Public License
  57 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  58 )
  59
  60 var (
  61         root             = flag.String("root", "./packages", "Path to packages directory")
  62         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  63         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  64         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  65         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  66         auth             = flag.String("auth", "passwd", "Path to file with login:passwd")
  67         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  68         version          = flag.Bool("version", false, "Print version information")
  69         warranty         = flag.Bool("warranty", false, "Print warranty information")
  70
  71         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  72         Version string = "UNKNOWN"
  73
  74         passwords map[string]string = make(map[string]string)
  75 )
  76
  77 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  78         path := filepath.Join(*root, dir)
  79         if _, err := os.Stat(path); os.IsNotExist(err) {
  80                 if err = os.Mkdir(path, 0700); err != nil {
  81                         http.Error(w, err.Error(), http.StatusInternalServerError)
  82                         return false
  83                 }
  84                 log.Println(r.RemoteAddr, "mkdir", dir)
  85         }
  86         return true
  87 }
  88
  89 func refreshDir(w http.ResponseWriter, r *http.Request, dir, filenameGet string) bool {
  90         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
  91                 log.Println(r.RemoteAddr, "pypi refresh skip, internal package", dir)
  92                 return true
  93         }
  94         log.Println(r.RemoteAddr, "pypi refresh", dir)
  95         resp, err := http.Get(*pypiURL + dir + "/")
  96         if err != nil {
  97                 http.Error(w, err.Error(), http.StatusBadGateway)
  98                 return false
  99         }
 100         defer resp.Body.Close()
 101         body, err := ioutil.ReadAll(resp.Body)
 102         if err != nil {
 103                 http.Error(w, err.Error(), http.StatusBadGateway)
 104                 return false
 105         }
 106         if !mkdirForPkg(w, r, dir) {
 107                 return false
 108         }
 109         var submatches []string
 110         var uri string
 111         var filename string
 112         var path string
 113         var pkgURL *url.URL
 114         var digest []byte
 115         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 116                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 117                 if len(submatches) == 0 {
 118                         continue
 119                 }
 120                 uri = submatches[1]
 121                 filename = submatches[2]
 122                 if pkgURL, err = url.Parse(uri); err != nil {
 123                         http.Error(w, err.Error(), http.StatusInternalServerError)
 124                         return false
 125                 }
 126                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 127                 if err != nil {
 128                         http.Error(w, err.Error(), http.StatusBadGateway)
 129                         return false
 130                 }
 131                 if filename == filenameGet {
 132                         log.Println(r.RemoteAddr, "pypi download", filename)
 133                         path = filepath.Join(*root, dir, filename)
 134                         resp, err = http.Get(uri)
 135                         if err != nil {
 136                                 http.Error(w, err.Error(), http.StatusBadGateway)
 137                                 return false
 138                         }
 139                         defer resp.Body.Close()
 140                         hasher := sha256.New()
 141                         dst, err := ioutil.TempFile(filepath.Join(*root, dir), "")
 142                         if err != nil {
 143                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 144                                 return false
 145                         }
 146                         wr := io.MultiWriter(hasher, dst)
 147                         if _, err = io.Copy(wr, resp.Body); err != nil {
 148                                 os.Remove(dst.Name())
 149                                 dst.Close()
 150                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 151                                 return false
 152                         }
 153                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 154                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 155                                 os.Remove(dst.Name())
 156                                 dst.Close()
 157                                 http.Error(w, err.Error(), http.StatusBadGateway)
 158                                 return false
 159                         }
 160                         if err = dst.Sync(); err != nil {
 161                                 os.Remove(dst.Name())
 162                                 dst.Close()
 163                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 164                                 return false
 165                         }
 166                         dst.Close()
 167                         if err = os.Rename(dst.Name(), path); err != nil {
 168                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 169                                 return false
 170                         }
 171                 }
 172                 path = filepath.Join(*root, dir, filename+SHA256Ext)
 173                 _, err = os.Stat(path)
 174                 if err == nil {
 175                         continue
 176                 } else {
 177                         if !os.IsNotExist(err) {
 178                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 179                                 return false
 180                         }
 181                 }
 182                 log.Println(r.RemoteAddr, "pypi touch", filename)
 183                 if err = ioutil.WriteFile(path, digest, os.FileMode(0600)); err != nil {
 184                         http.Error(w, err.Error(), http.StatusInternalServerError)
 185                         return false
 186                 }
 187         }
 188         return true
 189 }
 190
 191 func listRoot(w http.ResponseWriter, r *http.Request) {
 192         log.Println(r.RemoteAddr, "root")
 193         files, err := ioutil.ReadDir(*root)
 194         if err != nil {
 195                 http.Error(w, err.Error(), http.StatusInternalServerError)
 196                 return
 197         }
 198         w.Write([]byte(fmt.Sprintf(HTMLBegin, "root", "root")))
 199         for _, file := range files {
 200                 if file.Mode().IsDir() {
 201                         w.Write([]byte(fmt.Sprintf(
 202                                 HTMLElement,
 203                                 *refreshURLPath+file.Name()+"/",
 204                                 file.Name(),
 205                         )))
 206                 }
 207         }
 208         w.Write([]byte(HTMLEnd))
 209 }
 210
 211 func listDir(w http.ResponseWriter, r *http.Request, dir string, autorefresh bool) {
 212         log.Println(r.RemoteAddr, "dir", dir)
 213         dirPath := filepath.Join(*root, dir)
 214         if autorefresh {
 215                 if !refreshDir(w, r, dir, "") {
 216                         return
 217                 }
 218         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "") {
 219                 return
 220         }
 221         files, err := ioutil.ReadDir(dirPath)
 222         if err != nil {
 223                 http.Error(w, err.Error(), http.StatusInternalServerError)
 224                 return
 225         }
 226         w.Write([]byte(fmt.Sprintf(HTMLBegin, dir, dir)))
 227         var data []byte
 228         var filenameClean string
 229         for _, file := range files {
 230                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 231                         continue
 232                 }
 233                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 234                 if err != nil {
 235                         http.Error(w, err.Error(), http.StatusInternalServerError)
 236                         return
 237                 }
 238                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 239                 w.Write([]byte(fmt.Sprintf(
 240                         HTMLElement,
 241                         strings.Join([]string{
 242                                 *refreshURLPath, dir, "/",
 243                                 filenameClean, "#", SHA256Prefix, string(data),
 244                         }, ""),
 245                         filenameClean,
 246                 )))
 247         }
 248         w.Write([]byte(HTMLEnd))
 249 }
 250
 251 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 252         log.Println(r.RemoteAddr, "pkg", filename)
 253         path := filepath.Join(*root, dir, filename)
 254         if _, err := os.Stat(path); os.IsNotExist(err) {
 255                 if !refreshDir(w, r, dir, filename) {
 256                         return
 257                 }
 258         }
 259         http.ServeFile(w, r, path)
 260 }
 261
 262 func serveUpload(w http.ResponseWriter, r *http.Request) {
 263         username, password, ok := r.BasicAuth()
 264         if !ok || passwords[username] != password {
 265                 log.Println(r.RemoteAddr, "unauthenticated", username)
 266                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 267                 return
 268         }
 269         var err error
 270         if err = r.ParseMultipartForm(1 << 20); err != nil {
 271                 http.Error(w, err.Error(), http.StatusBadRequest)
 272                 return
 273         }
 274         for _, file := range r.MultipartForm.File["content"] {
 275                 filename := file.Filename
 276                 log.Println(r.RemoteAddr, "upload", filename, "by", username)
 277                 dir := filename[:strings.LastIndex(filename, "-")]
 278                 dirPath := filepath.Join(*root, dir)
 279                 path := filepath.Join(dirPath, filename)
 280                 if _, err = os.Stat(path); err == nil {
 281                         log.Println(r.RemoteAddr, "already exists", filename)
 282                         http.Error(w, "Already exists", http.StatusBadRequest)
 283                         return
 284                 }
 285                 if !mkdirForPkg(w, r, dir) {
 286                         return
 287                 }
 288                 internalPath := filepath.Join(dirPath, InternalFlag)
 289                 var dst *os.File
 290                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 291                         if dst, err = os.Create(internalPath); err != nil {
 292                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 293                                 return
 294                         }
 295                         dst.Close()
 296                 }
 297                 src, err := file.Open()
 298                 defer src.Close()
 299                 if err != nil {
 300                         http.Error(w, err.Error(), http.StatusInternalServerError)
 301                         return
 302                 }
 303                 dst, err = ioutil.TempFile(dirPath, "")
 304                 if err != nil {
 305                         http.Error(w, err.Error(), http.StatusInternalServerError)
 306                         return
 307                 }
 308                 hasher := sha256.New()
 309                 wr := io.MultiWriter(hasher, dst)
 310                 if _, err = io.Copy(wr, src); err != nil {
 311                         os.Remove(dst.Name())
 312                         dst.Close()
 313                         http.Error(w, err.Error(), http.StatusInternalServerError)
 314                         return
 315                 }
 316                 if err = dst.Sync(); err != nil {
 317                         os.Remove(dst.Name())
 318                         dst.Close()
 319                         http.Error(w, err.Error(), http.StatusInternalServerError)
 320                         return
 321                 }
 322                 dst.Close()
 323                 if err = os.Rename(dst.Name(), path); err != nil {
 324                         http.Error(w, err.Error(), http.StatusInternalServerError)
 325                         return
 326                 }
 327                 if err = ioutil.WriteFile(
 328                         path+SHA256Ext,
 329                         hasher.Sum(nil),
 330                         os.FileMode(0600),
 331                 ); err != nil {
 332                         http.Error(w, err.Error(), http.StatusInternalServerError)
 333                         return
 334                 }
 335         }
 336 }
 337
 338 func handler(w http.ResponseWriter, r *http.Request) {
 339         if r.Method == "GET" {
 340                 var path string
 341                 var autorefresh bool
 342                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 343                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 344                         autorefresh = false
 345                 } else {
 346                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 347                         autorefresh = true
 348                 }
 349                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 350                 if len(parts) > 2 {
 351                         http.Error(w, "invalid path", http.StatusBadRequest)
 352                         return
 353                 }
 354                 if len(parts) == 1 {
 355                         if parts[0] == "" {
 356                                 listRoot(w, r)
 357                         } else {
 358                                 listDir(w, r, parts[0], autorefresh)
 359                         }
 360                 } else {
 361                         servePkg(w, r, parts[0], parts[1])
 362                 }
 363         } else if r.Method == "POST" {
 364                 serveUpload(w, r)
 365         }
 366 }
 367
 368 func goodIntegrity() bool {
 369         dirs, err := ioutil.ReadDir(*root)
 370         if err != nil {
 371                 log.Fatal(err)
 372         }
 373         hasher := sha256.New()
 374         digest := make([]byte, sha256.Size)
 375         isGood := true
 376         var data []byte
 377         var pkgName string
 378         for _, dir := range dirs {
 379                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 380                 if err != nil {
 381                         log.Fatal(err)
 382                 }
 383                 for _, file := range files {
 384                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 385                                 continue
 386                         }
 387                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 388                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 389                         if err != nil {
 390                                 if os.IsNotExist(err) {
 391                                         continue
 392                                 }
 393                                 log.Fatal(err)
 394                         }
 395                         hasher.Write(data)
 396                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 397                         if err != nil {
 398                                 log.Fatal(err)
 399                         }
 400                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 401                                 log.Println(pkgName, "GOOD")
 402                         } else {
 403                                 isGood = false
 404                                 log.Println(pkgName, "BAD")
 405                         }
 406                         hasher.Reset()
 407                 }
 408         }
 409         return isGood
 410 }
 411
 412 func main() {
 413         flag.Parse()
 414         if *warranty {
 415                 fmt.Println(Warranty)
 416                 return
 417         }
 418         if *version {
 419                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 420                 return
 421         }
 422         if *fsck {
 423                 if !goodIntegrity() {
 424                         os.Exit(1)
 425                 }
 426                 return
 427         }
 428         auth, err := ioutil.ReadFile(*auth)
 429         if err != nil {
 430                 log.Fatal(err)
 431         }
 432         for _, credentials := range strings.Split(strings.TrimRight(string(auth), "\n"), "\n") {
 433                 splitted := strings.Split(credentials, ":")
 434                 if len(splitted) != 2 {
 435                         log.Fatal("Wrong auth format")
 436                 }
 437                 passwords[splitted[0]] = splitted[1]
 438         }
 439         log.Println("root:", *root, "bind:", *bind)
 440         http.HandleFunc(*norefreshURLPath, handler)
 441         http.HandleFunc(*refreshURLPath, handler)
 442         log.Fatal(http.ListenAndServe(*bind, nil))
 443 }