refresh.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bufio"
  22         "bytes"
  23         "crypto/md5"
  24         "crypto/sha256"
  25         "crypto/sha512"
  26         "encoding/hex"
  27         "hash"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "path/filepath"
  35         "strings"
  36
  37         "golang.org/x/crypto/blake2b"
  38 )
  39
  40 var pypiHTTPTransport http.Transport
  41
  42 func blake2b256New() hash.Hash {
  43         h, err := blake2b.New256(nil)
  44         if err != nil {
  45                 panic(err)
  46         }
  47         return h
  48 }
  49
  50 func agentedReq(url string) *http.Request {
  51         req, err := http.NewRequest("GET", url, nil)
  52         if err != nil {
  53                 log.Fatalln(err)
  54         }
  55         req.Header.Set("User-Agent", UserAgent)
  56         return req
  57 }
  58
  59 func refreshDir(
  60         w http.ResponseWriter,
  61         r *http.Request,
  62         pkgName, filenameGet string,
  63         gpgUpdate bool,
  64 ) bool {
  65         if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil {
  66                 return true
  67         }
  68         c := http.Client{Transport: &pypiHTTPTransport}
  69         resp, err := c.Get(*pypiURL + pkgName + "/")
  70         if err != nil {
  71                 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
  72                 http.Error(w, err.Error(), http.StatusBadGateway)
  73                 return false
  74         }
  75         if resp.StatusCode != http.StatusOK {
  76                 resp.Body.Close()
  77                 log.Println("error", r.RemoteAddr, "refresh", pkgName, "HTTP status:", resp.Status)
  78                 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
  79                 return false
  80         }
  81         body, err := ioutil.ReadAll(resp.Body)
  82         resp.Body.Close()
  83         if err != nil {
  84                 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
  85                 http.Error(w, err.Error(), http.StatusBadGateway)
  86                 return false
  87         }
  88         if !mkdirForPkg(w, r, pkgName) {
  89                 return false
  90         }
  91         dirPath := filepath.Join(*root, pkgName)
  92         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
  93                 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
  94                 if len(submatches) == 0 {
  95                         continue
  96                 }
  97                 uri := submatches[1]
  98                 filename := submatches[2]
  99                 pkgURL, err := url.Parse(uri)
 100                 if err != nil {
 101                         log.Println("error", r.RemoteAddr, "refresh", uri, err)
 102                         http.Error(w, err.Error(), http.StatusBadGateway)
 103                         return false
 104                 }
 105
 106                 if pkgURL.Fragment == "" {
 107                         log.Println(r.RemoteAddr, "pypi", filename, "no digest")
 108                         http.Error(w, "no digest provided", http.StatusBadGateway)
 109                         return false
 110                 }
 111                 digestInfo := strings.Split(pkgURL.Fragment, "=")
 112                 if len(digestInfo) == 1 {
 113                         // Ancient non PEP-0503 PyPIs, assume MD5
 114                         digestInfo = []string{"md5", digestInfo[0]}
 115                 } else if len(digestInfo) != 2 {
 116                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
 117                         http.Error(w, "invalid digest provided", http.StatusBadGateway)
 118                         return false
 119                 }
 120                 digest, err := hex.DecodeString(digestInfo[1])
 121                 if err != nil {
 122                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
 123                         http.Error(w, err.Error(), http.StatusBadGateway)
 124                         return false
 125                 }
 126                 hashAlgo := digestInfo[0]
 127                 var hasherNew func() hash.Hash
 128                 var hashSize int
 129                 switch hashAlgo {
 130                 case HashAlgoMD5:
 131                         hasherNew = md5.New
 132                         hashSize = md5.Size
 133                 case HashAlgoSHA256:
 134                         hasherNew = sha256.New
 135                         hashSize = sha256.Size
 136                 case HashAlgoSHA512:
 137                         hasherNew = sha512.New
 138                         hashSize = sha512.Size
 139                 case HashAlgoBLAKE2b256:
 140                         hasherNew = blake2b256New
 141                         hashSize = blake2b.Size256
 142                 default:
 143                         log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo)
 144                         http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
 145                         return false
 146                 }
 147                 if len(digest) != hashSize {
 148                         log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length")
 149                         http.Error(w, "invalid digest length", http.StatusBadGateway)
 150                         return false
 151                 }
 152
 153                 pkgURL.Fragment = ""
 154                 if pkgURL.Host == "" {
 155                         uri = pypiURLParsed.ResolveReference(pkgURL).String()
 156                 } else {
 157                         uri = pkgURL.String()
 158                 }
 159
 160                 path := filepath.Join(dirPath, filename)
 161                 if filename == filenameGet {
 162                         if killed {
 163                                 // Skip heavy remote call, when shutting down
 164                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 165                                 return false
 166                         }
 167                         log.Println(r.RemoteAddr, "pypi", filename, "download")
 168                         resp, err = c.Do(agentedReq(uri))
 169                         if err != nil {
 170                                 log.Println("error", r.RemoteAddr, "pypi", filename, "download", err)
 171                                 http.Error(w, err.Error(), http.StatusBadGateway)
 172                                 return false
 173                         }
 174                         defer resp.Body.Close()
 175                         if resp.StatusCode != http.StatusOK {
 176                                 log.Println(
 177                                         "error", r.RemoteAddr,
 178                                         "pypi", filename, "download",
 179                                         "HTTP status:", resp.Status,
 180                                 )
 181                                 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
 182                                 return false
 183                         }
 184                         hasher := hasherNew()
 185                         hasherSHA256 := sha256.New()
 186                         dst, err := TempFile(dirPath)
 187                         if err != nil {
 188                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 189                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 190                                 return false
 191                         }
 192                         dstBuf := bufio.NewWriter(dst)
 193                         wrs := []io.Writer{hasher, dstBuf}
 194                         if hashAlgo != HashAlgoSHA256 {
 195                                 wrs = append(wrs, hasherSHA256)
 196                         }
 197                         wr := io.MultiWriter(wrs...)
 198                         if _, err = io.Copy(wr, resp.Body); err != nil {
 199                                 os.Remove(dst.Name())
 200                                 dst.Close()
 201                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 202                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 203                                 return false
 204                         }
 205                         if err = dstBuf.Flush(); err != nil {
 206                                 os.Remove(dst.Name())
 207                                 dst.Close()
 208                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 209                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 210                                 return false
 211                         }
 212                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 213                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 214                                 os.Remove(dst.Name())
 215                                 dst.Close()
 216                                 http.Error(w, "digest mismatch", http.StatusBadGateway)
 217                                 return false
 218                         }
 219                         if !NoSync {
 220                                 if err = dst.Sync(); err != nil {
 221                                         os.Remove(dst.Name())
 222                                         dst.Close()
 223                                         log.Println("error", r.RemoteAddr, "pypi", filename, err)
 224                                         http.Error(w, err.Error(), http.StatusInternalServerError)
 225                                         return false
 226                                 }
 227                         }
 228                         if err = dst.Close(); err != nil {
 229                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 230                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 231                                 return false
 232                         }
 233                         if err = os.Rename(dst.Name(), path); err != nil {
 234                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 235                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 236                                 return false
 237                         }
 238                         if err = DirSync(dirPath); err != nil {
 239                                 log.Println("error", r.RemoteAddr, "pypi", filename, err)
 240                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 241                                 return false
 242                         }
 243                         if hashAlgo != HashAlgoSHA256 {
 244                                 hashAlgo = HashAlgoSHA256
 245                                 digest = hasherSHA256.Sum(nil)
 246                                 for _, algo := range knownHashAlgos[1:] {
 247                                         os.Remove(path + "." + algo)
 248                                 }
 249                         }
 250                 }
 251                 if filename == filenameGet || gpgUpdate {
 252                         if _, err = os.Stat(path); err != nil {
 253                                 goto GPGSigSkip
 254                         }
 255                         resp, err := c.Do(agentedReq(uri + GPGSigExt))
 256                         if err != nil {
 257                                 goto GPGSigSkip
 258                         }
 259                         if resp.StatusCode != http.StatusOK {
 260                                 resp.Body.Close()
 261                                 goto GPGSigSkip
 262                         }
 263                         sig, err := ioutil.ReadAll(resp.Body)
 264                         resp.Body.Close()
 265                         if err != nil {
 266                                 goto GPGSigSkip
 267                         }
 268                         if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
 269                                 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
 270                                 goto GPGSigSkip
 271                         }
 272                         if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
 273                                 log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
 274                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 275                                 return false
 276                         }
 277                         log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
 278                 }
 279         GPGSigSkip:
 280                 path = path + "." + hashAlgo
 281                 _, err = os.Stat(path)
 282                 if err == nil {
 283                         continue
 284                 }
 285                 if !os.IsNotExist(err) {
 286                         log.Println("error", r.RemoteAddr, "pypi", path, err)
 287                         http.Error(w, err.Error(), http.StatusInternalServerError)
 288                         return false
 289                 }
 290                 log.Println(r.RemoteAddr, "pypi", filename, "touch")
 291                 if err = WriteFileSync(dirPath, path, digest); err != nil {
 292                         log.Println("error", r.RemoteAddr, "pypi", path, err)
 293                         http.Error(w, err.Error(), http.StatusInternalServerError)
 294                         return false
 295                 }
 296         }
 297         return true
 298 }