refresh.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2020 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bufio"
  22         "bytes"
  23         "crypto/md5"
  24         "crypto/sha256"
  25         "crypto/sha512"
  26         "encoding/hex"
  27         "hash"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "path/filepath"
  35         "strings"
  36
  37         "golang.org/x/crypto/blake2b"
  38 )
  39
  40 func blake2b256New() hash.Hash {
  41         h, err := blake2b.New256(nil)
  42         if err != nil {
  43                 panic(err)
  44         }
  45         return h
  46 }
  47
  48 func refreshDir(
  49         w http.ResponseWriter,
  50         r *http.Request,
  51         pkgName, filenameGet string,
  52         gpgUpdate bool,
  53 ) bool {
  54         if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil {
  55                 return true
  56         }
  57         resp, err := http.Get(*pypiURL + pkgName + "/")
  58         if err != nil {
  59                 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
  60                 http.Error(w, err.Error(), http.StatusBadGateway)
  61                 return false
  62         }
  63         if resp.StatusCode != http.StatusOK {
  64                 resp.Body.Close()
  65                 log.Println("error", r.RemoteAddr, "refresh", pkgName, "HTTP status:", resp.Status)
  66                 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
  67                 return false
  68         }
  69         body, err := ioutil.ReadAll(resp.Body)
  70         resp.Body.Close()
  71         if err != nil {
  72                 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
  73                 http.Error(w, err.Error(), http.StatusBadGateway)
  74                 return false
  75         }
  76         if !mkdirForPkg(w, r, pkgName) {
  77                 return false
  78         }
  79         dirPath := filepath.Join(*root, pkgName)
  80         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
  81                 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
  82                 if len(submatches) == 0 {
  83                         continue
  84                 }
  85                 uri := submatches[1]
  86                 filename := submatches[2]
  87                 pkgURL, err := url.Parse(uri)
  88                 if err != nil {
  89                         log.Println("error", r.RemoteAddr, "refresh", uri, err)
  90                         http.Error(w, err.Error(), http.StatusBadGateway)
  91                         return false
  92                 }
  93
  94                 if pkgURL.Fragment == "" {
  95                         log.Println(r.RemoteAddr, "pypi", filename, "no digest")
  96                         http.Error(w, "no digest provided", http.StatusBadGateway)
  97                         return false
  98                 }
  99                 digestInfo := strings.Split(pkgURL.Fragment, "=")
 100                 if len(digestInfo) == 1 {
 101                         // Ancient non PEP-0503 PyPIs, assume MD5
 102                         digestInfo = []string{"md5", digestInfo[0]}
 103                 } else if len(digestInfo) != 2 {
 104                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
 105                         http.Error(w, "invalid digest provided", http.StatusBadGateway)
 106                         return false
 107                 }
 108                 digest, err := hex.DecodeString(digestInfo[1])
 109                 if err != nil {
 110                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
 111                         http.Error(w, err.Error(), http.StatusBadGateway)
 112                         return false
 113                 }
 114                 hashAlgo := digestInfo[0]
 115                 var hasherNew func() hash.Hash
 116                 var hashSize int
 117                 switch hashAlgo {
 118                 case HashAlgoMD5:
 119                         hasherNew = md5.New
 120                         hashSize = md5.Size
 121                 case HashAlgoSHA256:
 122                         hasherNew = sha256.New
 123                         hashSize = sha256.Size
 124                 case HashAlgoSHA512:
 125                         hasherNew = sha512.New
 126                         hashSize = sha512.Size
 127                 case HashAlgoBLAKE2b256:
 128                         hasherNew = blake2b256New
 129                         hashSize = blake2b.Size256
 130                 default:
 131                         log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo)
 132                         http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
 133                         return false
 134                 }
 135                 if len(digest) != hashSize {
 136                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length")
 137                         http.Error(w, "invalid digest length", http.StatusBadGateway)
 138                         return false
 139                 }
 140
 141                 pkgURL.Fragment = ""
 142                 if pkgURL.Host == "" {
 143                         uri = pypiURLParsed.ResolveReference(pkgURL).String()
 144                 } else {
 145                         uri = pkgURL.String()
 146                 }
 147
 148                 path := filepath.Join(dirPath, filename)
 149                 if filename == filenameGet {
 150                         if killed {
 151                                 // Skip heavy remote call, when shutting down
 152                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 153                                 return false
 154                         }
 155                         log.Println(r.RemoteAddr, "pypi", filename, "download")
 156                         resp, err = http.Get(uri)
 157                         if err != nil {
 158                                 log.Println("error", r.RemoteAddr, "pypi", filename, "download", err)
 159                                 http.Error(w, err.Error(), http.StatusBadGateway)
 160                                 return false
 161                         }
 162                         defer resp.Body.Close()
 163                         if resp.StatusCode != http.StatusOK {
 164                                 log.Println(
 165                                         "error", r.RemoteAddr,
 166                                         "pypi", filename, "download",
 167                                         "HTTP status:", resp.Status,
 168                                 )
 169                                 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
 170                                 return false
 171                         }
 172                         hasher := hasherNew()
 173                         hasherSHA256 := sha256.New()
 174                         dst, err := TempFile(dirPath)
 175                         if err != nil {
 176                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 177                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 178                                 return false
 179                         }
 180                         dstBuf := bufio.NewWriter(dst)
 181                         wrs := []io.Writer{hasher, dstBuf}
 182                         if hashAlgo != HashAlgoSHA256 {
 183                                 wrs = append(wrs, hasherSHA256)
 184                         }
 185                         wr := io.MultiWriter(wrs...)
 186                         if _, err = io.Copy(wr, resp.Body); err != nil {
 187                                 os.Remove(dst.Name())
 188                                 dst.Close()
 189                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 190                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 191                                 return false
 192                         }
 193                         if err = dstBuf.Flush(); err != nil {
 194                                 os.Remove(dst.Name())
 195                                 dst.Close()
 196                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 197                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 198                                 return false
 199                         }
 200                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 201                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 202                                 os.Remove(dst.Name())
 203                                 dst.Close()
 204                                 http.Error(w, "digest mismatch", http.StatusBadGateway)
 205                                 return false
 206                         }
 207                         if err = dst.Sync(); err != nil {
 208                                 os.Remove(dst.Name())
 209                                 dst.Close()
 210                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 211                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 212                                 return false
 213                         }
 214                         if err = dst.Close(); err != nil {
 215                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 216                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 217                                 return false
 218                         }
 219                         if err = os.Rename(dst.Name(), path); err != nil {
 220                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 221                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 222                                 return false
 223                         }
 224                         if err = DirSync(dirPath); err != nil {
 225                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 226                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 227                                 return false
 228                         }
 229                         if hashAlgo != HashAlgoSHA256 {
 230                                 hashAlgo = HashAlgoSHA256
 231                                 digest = hasherSHA256.Sum(nil)
 232                                 for _, algo := range knownHashAlgos[1:] {
 233                                         os.Remove(path + "." + algo)
 234                                 }
 235                         }
 236                 }
 237                 if filename == filenameGet || gpgUpdate {
 238                         if _, err = os.Stat(path); err != nil {
 239                                 goto GPGSigSkip
 240                         }
 241                         resp, err := http.Get(uri + GPGSigExt)
 242                         if err != nil {
 243                                 goto GPGSigSkip
 244                         }
 245                         if resp.StatusCode != http.StatusOK {
 246                                 resp.Body.Close()
 247                                 goto GPGSigSkip
 248                         }
 249                         sig, err := ioutil.ReadAll(resp.Body)
 250                         resp.Body.Close()
 251                         if err != nil {
 252                                 goto GPGSigSkip
 253                         }
 254                         if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
 255                                 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
 256                                 goto GPGSigSkip
 257                         }
 258                         if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
 259                                 log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
 260                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 261                                 return false
 262                         }
 263                         log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
 264                 }
 265         GPGSigSkip:
 266                 path = path + "." + hashAlgo
 267                 _, err = os.Stat(path)
 268                 if err == nil {
 269                         continue
 270                 }
 271                 if !os.IsNotExist(err) {
 272                         log.Println("error", r.RemoteAddr, "pypi", path, err)
 273                         http.Error(w, err.Error(), http.StatusInternalServerError)
 274                         return false
 275                 }
 276                 log.Println(r.RemoteAddr, "pypi", filename, "touch")
 277                 if err = WriteFileSync(dirPath, path, digest); err != nil {
 278                         log.Println("error", r.RemoteAddr, "pypi", path, err)
 279                         http.Error(w, err.Error(), http.StatusInternalServerError)
 280                         return false
 281                 }
 282         }
 283         return true
 284 }