refresh.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2020 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bufio"
  22         "bytes"
  23         "crypto/md5"
  24         "crypto/sha256"
  25         "crypto/sha512"
  26         "encoding/hex"
  27         "hash"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "path/filepath"
  35         "strings"
  36
  37         "golang.org/x/crypto/blake2b"
  38 )
  39
  40 func blake2b256New() hash.Hash {
  41         h, err := blake2b.New256(nil)
  42         if err != nil {
  43                 panic(err)
  44         }
  45         return h
  46 }
  47
  48 func refreshDir(
  49         w http.ResponseWriter,
  50         r *http.Request,
  51         pkgName, filenameGet string,
  52         gpgUpdate bool,
  53 ) bool {
  54         if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil {
  55                 return true
  56         }
  57         resp, err := http.Get(*pypiURL + pkgName + "/")
  58         if err != nil {
  59                 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
  60                 http.Error(w, err.Error(), http.StatusBadGateway)
  61                 return false
  62         }
  63         body, err := ioutil.ReadAll(resp.Body)
  64         resp.Body.Close()
  65         if err != nil {
  66                 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
  67                 http.Error(w, err.Error(), http.StatusBadGateway)
  68                 return false
  69         }
  70         if !mkdirForPkg(w, r, pkgName) {
  71                 return false
  72         }
  73         dirPath := filepath.Join(*root, pkgName)
  74         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
  75                 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
  76                 if len(submatches) == 0 {
  77                         continue
  78                 }
  79                 uri := submatches[1]
  80                 filename := submatches[2]
  81                 pkgURL, err := url.Parse(uri)
  82                 if err != nil {
  83                         log.Println("error", r.RemoteAddr, "refresh", uri, err)
  84                         http.Error(w, err.Error(), http.StatusBadGateway)
  85                         return false
  86                 }
  87
  88                 if pkgURL.Fragment == "" {
  89                         log.Println(r.RemoteAddr, "pypi", filename, "no digest")
  90                         http.Error(w, "no digest provided", http.StatusBadGateway)
  91                         return false
  92                 }
  93                 digestInfo := strings.Split(pkgURL.Fragment, "=")
  94                 if len(digestInfo) == 1 {
  95                         // Ancient non PEP-0503 PyPIs, assume MD5
  96                         digestInfo = []string{"md5", digestInfo[0]}
  97                 } else if len(digestInfo) != 2 {
  98                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
  99                         http.Error(w, "invalid digest provided", http.StatusBadGateway)
 100                         return false
 101                 }
 102                 digest, err := hex.DecodeString(digestInfo[1])
 103                 if err != nil {
 104                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
 105                         http.Error(w, err.Error(), http.StatusBadGateway)
 106                         return false
 107                 }
 108                 hashAlgo := digestInfo[0]
 109                 var hasherNew func() hash.Hash
 110                 var hashSize int
 111                 switch hashAlgo {
 112                 case HashAlgoMD5:
 113                         hasherNew = md5.New
 114                         hashSize = md5.Size
 115                 case HashAlgoSHA256:
 116                         hasherNew = sha256.New
 117                         hashSize = sha256.Size
 118                 case HashAlgoSHA512:
 119                         hasherNew = sha512.New
 120                         hashSize = sha512.Size
 121                 case HashAlgoBLAKE2b256:
 122                         hasherNew = blake2b256New
 123                         hashSize = blake2b.Size256
 124                 default:
 125                         log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo)
 126                         http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
 127                         return false
 128                 }
 129                 if len(digest) != hashSize {
 130                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length")
 131                         http.Error(w, "invalid digest length", http.StatusBadGateway)
 132                         return false
 133                 }
 134
 135                 pkgURL.Fragment = ""
 136                 if pkgURL.Host == "" {
 137                         uri = pypiURLParsed.ResolveReference(pkgURL).String()
 138                 } else {
 139                         uri = pkgURL.String()
 140                 }
 141
 142                 path := filepath.Join(dirPath, filename)
 143                 if filename == filenameGet {
 144                         if killed {
 145                                 // Skip heavy remote call, when shutting down
 146                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 147                                 return false
 148                         }
 149                         log.Println(r.RemoteAddr, "pypi", filename, "download")
 150                         resp, err = http.Get(uri)
 151                         if err != nil {
 152                                 log.Println("error", r.RemoteAddr, "pypi", filename, "download", err)
 153                                 http.Error(w, err.Error(), http.StatusBadGateway)
 154                                 return false
 155                         }
 156                         defer resp.Body.Close()
 157                         hasher := hasherNew()
 158                         hasherSHA256 := sha256.New()
 159                         dst, err := TempFile(dirPath)
 160                         if err != nil {
 161                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 162                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 163                                 return false
 164                         }
 165                         dstBuf := bufio.NewWriter(dst)
 166                         wrs := []io.Writer{hasher, dstBuf}
 167                         if hashAlgo != HashAlgoSHA256 {
 168                                 wrs = append(wrs, hasherSHA256)
 169                         }
 170                         wr := io.MultiWriter(wrs...)
 171                         if _, err = io.Copy(wr, resp.Body); err != nil {
 172                                 os.Remove(dst.Name())
 173                                 dst.Close()
 174                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 175                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 176                                 return false
 177                         }
 178                         if err = dstBuf.Flush(); err != nil {
 179                                 os.Remove(dst.Name())
 180                                 dst.Close()
 181                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 182                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 183                                 return false
 184                         }
 185                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 186                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 187                                 os.Remove(dst.Name())
 188                                 dst.Close()
 189                                 http.Error(w, "digest mismatch", http.StatusBadGateway)
 190                                 return false
 191                         }
 192                         if err = dst.Sync(); err != nil {
 193                                 os.Remove(dst.Name())
 194                                 dst.Close()
 195                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 196                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 197                                 return false
 198                         }
 199                         if err = dst.Close(); err != nil {
 200                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 201                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 202                                 return false
 203                         }
 204                         if err = os.Rename(dst.Name(), path); err != nil {
 205                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 206                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 207                                 return false
 208                         }
 209                         if err = DirSync(dirPath); err != nil {
 210                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 211                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 212                                 return false
 213                         }
 214                         if hashAlgo != HashAlgoSHA256 {
 215                                 hashAlgo = HashAlgoSHA256
 216                                 digest = hasherSHA256.Sum(nil)
 217                                 for _, algo := range knownHashAlgos[1:] {
 218                                         os.Remove(path + "." + algo)
 219                                 }
 220                         }
 221                 }
 222                 if filename == filenameGet || gpgUpdate {
 223                         if _, err = os.Stat(path); err != nil {
 224                                 goto GPGSigSkip
 225                         }
 226                         resp, err := http.Get(uri + GPGSigExt)
 227                         if err != nil {
 228                                 goto GPGSigSkip
 229                         }
 230                         if resp.StatusCode != http.StatusOK {
 231                                 resp.Body.Close()
 232                                 goto GPGSigSkip
 233                         }
 234                         sig, err := ioutil.ReadAll(resp.Body)
 235                         resp.Body.Close()
 236                         if err != nil {
 237                                 goto GPGSigSkip
 238                         }
 239                         if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
 240                                 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
 241                                 goto GPGSigSkip
 242                         }
 243                         if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
 244                                 log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
 245                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 246                                 return false
 247                         }
 248                         log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
 249                 }
 250         GPGSigSkip:
 251                 path = path + "." + hashAlgo
 252                 _, err = os.Stat(path)
 253                 if err == nil {
 254                         continue
 255                 }
 256                 if !os.IsNotExist(err) {
 257                         log.Println("error", r.RemoteAddr, "pypi", path, err)
 258                         http.Error(w, err.Error(), http.StatusInternalServerError)
 259                         return false
 260                 }
 261                 log.Println(r.RemoteAddr, "pypi", filename, "touch")
 262                 if err = WriteFileSync(dirPath, path, digest); err != nil {
 263                         log.Println("error", r.RemoteAddr, "pypi", path, err)
 264                         http.Error(w, err.Error(), http.StatusInternalServerError)
 265                         return false
 266                 }
 267         }
 268         return true
 269 }