refresh.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bufio"
  22         "bytes"
  23         "crypto/md5"
  24         "crypto/sha256"
  25         "crypto/sha512"
  26         "encoding/hex"
  27         "hash"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "path/filepath"
  35         "strings"
  36
  37         "golang.org/x/crypto/blake2b"
  38 )
  39
  40 var pypiHTTPTransport http.Transport
  41
  42 func blake2b256New() hash.Hash {
  43         h, err := blake2b.New256(nil)
  44         if err != nil {
  45                 panic(err)
  46         }
  47         return h
  48 }
  49
  50 func refreshDir(
  51         w http.ResponseWriter,
  52         r *http.Request,
  53         pkgName, filenameGet string,
  54         gpgUpdate bool,
  55 ) bool {
  56         if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil {
  57                 return true
  58         }
  59         c := http.Client{Transport: &pypiHTTPTransport}
  60         resp, err := c.Get(*pypiURL + pkgName + "/")
  61         if err != nil {
  62                 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
  63                 http.Error(w, err.Error(), http.StatusBadGateway)
  64                 return false
  65         }
  66         if resp.StatusCode != http.StatusOK {
  67                 resp.Body.Close()
  68                 log.Println("error", r.RemoteAddr, "refresh", pkgName, "HTTP status:", resp.Status)
  69                 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
  70                 return false
  71         }
  72         body, err := ioutil.ReadAll(resp.Body)
  73         resp.Body.Close()
  74         if err != nil {
  75                 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
  76                 http.Error(w, err.Error(), http.StatusBadGateway)
  77                 return false
  78         }
  79         if !mkdirForPkg(w, r, pkgName) {
  80                 return false
  81         }
  82         dirPath := filepath.Join(*root, pkgName)
  83         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
  84                 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
  85                 if len(submatches) == 0 {
  86                         continue
  87                 }
  88                 uri := submatches[1]
  89                 filename := submatches[2]
  90                 pkgURL, err := url.Parse(uri)
  91                 if err != nil {
  92                         log.Println("error", r.RemoteAddr, "refresh", uri, err)
  93                         http.Error(w, err.Error(), http.StatusBadGateway)
  94                         return false
  95                 }
  96
  97                 if pkgURL.Fragment == "" {
  98                         log.Println(r.RemoteAddr, "pypi", filename, "no digest")
  99                         http.Error(w, "no digest provided", http.StatusBadGateway)
 100                         return false
 101                 }
 102                 digestInfo := strings.Split(pkgURL.Fragment, "=")
 103                 if len(digestInfo) == 1 {
 104                         // Ancient non PEP-0503 PyPIs, assume MD5
 105                         digestInfo = []string{"md5", digestInfo[0]}
 106                 } else if len(digestInfo) != 2 {
 107                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
 108                         http.Error(w, "invalid digest provided", http.StatusBadGateway)
 109                         return false
 110                 }
 111                 digest, err := hex.DecodeString(digestInfo[1])
 112                 if err != nil {
 113                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
 114                         http.Error(w, err.Error(), http.StatusBadGateway)
 115                         return false
 116                 }
 117                 hashAlgo := digestInfo[0]
 118                 var hasherNew func() hash.Hash
 119                 var hashSize int
 120                 switch hashAlgo {
 121                 case HashAlgoMD5:
 122                         hasherNew = md5.New
 123                         hashSize = md5.Size
 124                 case HashAlgoSHA256:
 125                         hasherNew = sha256.New
 126                         hashSize = sha256.Size
 127                 case HashAlgoSHA512:
 128                         hasherNew = sha512.New
 129                         hashSize = sha512.Size
 130                 case HashAlgoBLAKE2b256:
 131                         hasherNew = blake2b256New
 132                         hashSize = blake2b.Size256
 133                 default:
 134                         log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo)
 135                         http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
 136                         return false
 137                 }
 138                 if len(digest) != hashSize {
 139                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length")
 140                         http.Error(w, "invalid digest length", http.StatusBadGateway)
 141                         return false
 142                 }
 143
 144                 pkgURL.Fragment = ""
 145                 if pkgURL.Host == "" {
 146                         uri = pypiURLParsed.ResolveReference(pkgURL).String()
 147                 } else {
 148                         uri = pkgURL.String()
 149                 }
 150
 151                 path := filepath.Join(dirPath, filename)
 152                 if filename == filenameGet {
 153                         if killed {
 154                                 // Skip heavy remote call, when shutting down
 155                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 156                                 return false
 157                         }
 158                         log.Println(r.RemoteAddr, "pypi", filename, "download")
 159                         resp, err = http.Get(uri)
 160                         if err != nil {
 161                                 log.Println("error", r.RemoteAddr, "pypi", filename, "download", err)
 162                                 http.Error(w, err.Error(), http.StatusBadGateway)
 163                                 return false
 164                         }
 165                         defer resp.Body.Close()
 166                         if resp.StatusCode != http.StatusOK {
 167                                 log.Println(
 168                                         "error", r.RemoteAddr,
 169                                         "pypi", filename, "download",
 170                                         "HTTP status:", resp.Status,
 171                                 )
 172                                 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
 173                                 return false
 174                         }
 175                         hasher := hasherNew()
 176                         hasherSHA256 := sha256.New()
 177                         dst, err := TempFile(dirPath)
 178                         if err != nil {
 179                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 180                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 181                                 return false
 182                         }
 183                         dstBuf := bufio.NewWriter(dst)
 184                         wrs := []io.Writer{hasher, dstBuf}
 185                         if hashAlgo != HashAlgoSHA256 {
 186                                 wrs = append(wrs, hasherSHA256)
 187                         }
 188                         wr := io.MultiWriter(wrs...)
 189                         if _, err = io.Copy(wr, resp.Body); err != nil {
 190                                 os.Remove(dst.Name())
 191                                 dst.Close()
 192                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 193                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 194                                 return false
 195                         }
 196                         if err = dstBuf.Flush(); err != nil {
 197                                 os.Remove(dst.Name())
 198                                 dst.Close()
 199                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 200                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 201                                 return false
 202                         }
 203                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 204                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 205                                 os.Remove(dst.Name())
 206                                 dst.Close()
 207                                 http.Error(w, "digest mismatch", http.StatusBadGateway)
 208                                 return false
 209                         }
 210                         if err = dst.Sync(); err != nil {
 211                                 os.Remove(dst.Name())
 212                                 dst.Close()
 213                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 214                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 215                                 return false
 216                         }
 217                         if err = dst.Close(); err != nil {
 218                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 219                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 220                                 return false
 221                         }
 222                         if err = os.Rename(dst.Name(), path); err != nil {
 223                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 224                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 225                                 return false
 226                         }
 227                         if err = DirSync(dirPath); err != nil {
 228                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 229                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 230                                 return false
 231                         }
 232                         if hashAlgo != HashAlgoSHA256 {
 233                                 hashAlgo = HashAlgoSHA256
 234                                 digest = hasherSHA256.Sum(nil)
 235                                 for _, algo := range knownHashAlgos[1:] {
 236                                         os.Remove(path + "." + algo)
 237                                 }
 238                         }
 239                 }
 240                 if filename == filenameGet || gpgUpdate {
 241                         if _, err = os.Stat(path); err != nil {
 242                                 goto GPGSigSkip
 243                         }
 244                         resp, err := http.Get(uri + GPGSigExt)
 245                         if err != nil {
 246                                 goto GPGSigSkip
 247                         }
 248                         if resp.StatusCode != http.StatusOK {
 249                                 resp.Body.Close()
 250                                 goto GPGSigSkip
 251                         }
 252                         sig, err := ioutil.ReadAll(resp.Body)
 253                         resp.Body.Close()
 254                         if err != nil {
 255                                 goto GPGSigSkip
 256                         }
 257                         if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
 258                                 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
 259                                 goto GPGSigSkip
 260                         }
 261                         if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
 262                                 log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
 263                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 264                                 return false
 265                         }
 266                         log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
 267                 }
 268         GPGSigSkip:
 269                 path = path + "." + hashAlgo
 270                 _, err = os.Stat(path)
 271                 if err == nil {
 272                         continue
 273                 }
 274                 if !os.IsNotExist(err) {
 275                         log.Println("error", r.RemoteAddr, "pypi", path, err)
 276                         http.Error(w, err.Error(), http.StatusInternalServerError)
 277                         return false
 278                 }
 279                 log.Println(r.RemoteAddr, "pypi", filename, "touch")
 280                 if err = WriteFileSync(dirPath, path, digest); err != nil {
 281                         log.Println("error", r.RemoteAddr, "pypi", path, err)
 282                         http.Error(w, err.Error(), http.StatusInternalServerError)
 283                         return false
 284                 }
 285         }
 286         return true
 287 }