refresh.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bufio"
  22         "bytes"
  23         "crypto/md5"
  24         "crypto/sha256"
  25         "crypto/sha512"
  26         "encoding/hex"
  27         "hash"
  28         "io"
  29         "io/ioutil"
  30         "log"
  31         "net/http"
  32         "net/url"
  33         "os"
  34         "path/filepath"
  35         "strings"
  36
  37         "golang.org/x/crypto/blake2b"
  38 )
  39
  40 func blake2b256New() hash.Hash {
  41         h, err := blake2b.New256(nil)
  42         if err != nil {
  43                 panic(err)
  44         }
  45         return h
  46 }
  47
  48 func refreshDir(
  49         w http.ResponseWriter,
  50         r *http.Request,
  51         dir,
  52         filenameGet string,
  53         gpgUpdate bool,
  54 ) bool {
  55         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
  56                 return true
  57         }
  58         resp, err := http.Get(*pypiURL + dir + "/")
  59         if err != nil {
  60                 http.Error(w, err.Error(), http.StatusBadGateway)
  61                 return false
  62         }
  63         body, err := ioutil.ReadAll(resp.Body)
  64         resp.Body.Close()
  65         if err != nil {
  66                 http.Error(w, err.Error(), http.StatusBadGateway)
  67                 return false
  68         }
  69         if !mkdirForPkg(w, r, dir) {
  70                 return false
  71         }
  72         dirPath := filepath.Join(*root, dir)
  73         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
  74                 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
  75                 if len(submatches) == 0 {
  76                         continue
  77                 }
  78                 uri := submatches[1]
  79                 filename := submatches[2]
  80                 pkgURL, err := url.Parse(uri)
  81                 if err != nil {
  82                         http.Error(w, err.Error(), http.StatusBadGateway)
  83                         return false
  84                 }
  85
  86                 if pkgURL.Fragment == "" {
  87                         log.Println(r.RemoteAddr, "pypi", filename, "no digest provided")
  88                         http.Error(w, "no digest provided", http.StatusBadGateway)
  89                         return false
  90                 }
  91                 digestInfo := strings.Split(pkgURL.Fragment, "=")
  92                 if len(digestInfo) == 1 {
  93                         // Ancient non PEP-0503 PyPIs, assume MD5
  94                         digestInfo = []string{"md5", digestInfo[0]}
  95                 } else if len(digestInfo) != 2 {
  96                         log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided")
  97                         http.Error(w, "invalid digest provided", http.StatusBadGateway)
  98                         return false
  99                 }
 100                 digest, err := hex.DecodeString(digestInfo[1])
 101                 if err != nil {
 102                         http.Error(w, err.Error(), http.StatusBadGateway)
 103                         return false
 104                 }
 105                 hashAlgo := digestInfo[0]
 106                 var hasherNew func() hash.Hash
 107                 var hashSize int
 108                 switch hashAlgo {
 109                 case HashAlgoMD5:
 110                         hasherNew = md5.New
 111                         hashSize = md5.Size
 112                 case HashAlgoSHA256:
 113                         hasherNew = sha256.New
 114                         hashSize = sha256.Size
 115                 case HashAlgoSHA512:
 116                         hasherNew = sha512.New
 117                         hashSize = sha512.Size
 118                 case HashAlgoBLAKE2b256:
 119                         hasherNew = blake2b256New
 120                         hashSize = blake2b.Size256
 121                 default:
 122                         log.Println(
 123                                 r.RemoteAddr, "pypi", filename,
 124                                 "unknown digest algorithm", hashAlgo,
 125                         )
 126                         http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
 127                         return false
 128                 }
 129                 if len(digest) != hashSize {
 130                         log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length")
 131                         http.Error(w, "invalid digest length", http.StatusBadGateway)
 132                         return false
 133                 }
 134
 135                 pkgURL.Fragment = ""
 136                 if pkgURL.Host == "" {
 137                         uri = pypiURLParsed.ResolveReference(pkgURL).String()
 138                 } else {
 139                         uri = pkgURL.String()
 140                 }
 141
 142                 path := filepath.Join(dirPath, filename)
 143                 if filename == filenameGet {
 144                         if killed {
 145                                 // Skip heavy remote call, when shutting down
 146                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 147                                 return false
 148                         }
 149                         log.Println(r.RemoteAddr, "pypi download", filename)
 150                         resp, err = http.Get(uri)
 151                         if err != nil {
 152                                 log.Println(r.RemoteAddr, "pypi download error:", err.Error())
 153                                 http.Error(w, err.Error(), http.StatusBadGateway)
 154                                 return false
 155                         }
 156                         defer resp.Body.Close()
 157                         hasher := hasherNew()
 158                         hasherSHA256 := sha256.New()
 159                         dst, err := TempFile(dirPath)
 160                         if err != nil {
 161                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 162                                 return false
 163                         }
 164                         dstBuf := bufio.NewWriter(dst)
 165                         wrs := []io.Writer{hasher, dstBuf}
 166                         if hashAlgo != HashAlgoSHA256 {
 167                                 wrs = append(wrs, hasherSHA256)
 168                         }
 169                         wr := io.MultiWriter(wrs...)
 170                         if _, err = io.Copy(wr, resp.Body); err != nil {
 171                                 os.Remove(dst.Name())
 172                                 dst.Close()
 173                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 174                                 return false
 175                         }
 176                         if err = dstBuf.Flush(); err != nil {
 177                                 os.Remove(dst.Name())
 178                                 dst.Close()
 179                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 180                                 return false
 181                         }
 182                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 183                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 184                                 os.Remove(dst.Name())
 185                                 dst.Close()
 186                                 http.Error(w, "digest mismatch", http.StatusBadGateway)
 187                                 return false
 188                         }
 189                         if err = dst.Sync(); err != nil {
 190                                 os.Remove(dst.Name())
 191                                 dst.Close()
 192                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 193                                 return false
 194                         }
 195                         if err = dst.Close(); err != nil {
 196                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 197                                 return false
 198                         }
 199                         if err = os.Rename(dst.Name(), path); err != nil {
 200                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 201                                 return false
 202                         }
 203                         if err = DirSync(dirPath); err != nil {
 204                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 205                                 return false
 206                         }
 207                         if hashAlgo != HashAlgoSHA256 {
 208                                 hashAlgo = HashAlgoSHA256
 209                                 digest = hasherSHA256.Sum(nil)
 210                                 for _, algo := range knownHashAlgos[1:] {
 211                                         os.Remove(path + "." + algo)
 212                                 }
 213                         }
 214                 }
 215                 if filename == filenameGet || gpgUpdate {
 216                         if _, err = os.Stat(path); err != nil {
 217                                 goto GPGSigSkip
 218                         }
 219                         resp, err := http.Get(uri + GPGSigExt)
 220                         if err != nil {
 221                                 goto GPGSigSkip
 222                         }
 223                         if resp.StatusCode != http.StatusOK {
 224                                 resp.Body.Close()
 225                                 goto GPGSigSkip
 226                         }
 227                         sig, err := ioutil.ReadAll(resp.Body)
 228                         resp.Body.Close()
 229                         if err != nil {
 230                                 goto GPGSigSkip
 231                         }
 232                         if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
 233                                 log.Println(r.RemoteAddr, "pypi non PGP signature", filename)
 234                                 goto GPGSigSkip
 235                         }
 236                         if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
 237                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 238                                 return false
 239                         }
 240                         log.Println(r.RemoteAddr, "pypi downloaded signature", filename)
 241                 }
 242         GPGSigSkip:
 243                 path = path + "." + hashAlgo
 244                 _, err = os.Stat(path)
 245                 if err == nil {
 246                         continue
 247                 }
 248                 if !os.IsNotExist(err) {
 249                         http.Error(w, err.Error(), http.StatusInternalServerError)
 250                         return false
 251                 }
 252                 log.Println(r.RemoteAddr, "pypi touch", filename)
 253                 if err = WriteFileSync(dirPath, path, digest); err != nil {
 254                         http.Error(w, err.Error(), http.StatusInternalServerError)
 255                         return false
 256                 }
 257         }
 258         return true
 259 }