2 GoCheese -- Python private package repository and caching proxy
3 Copyright (C) 2019-2020 Sergey Matveev <stargrave@stargrave.org>
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>.
37 "golang.org/x/crypto/blake2b"
40 func blake2b256New() hash.Hash {
41 h, err := blake2b.New256(nil)
49 w http.ResponseWriter,
51 pkgName, filenameGet string,
54 if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil {
57 resp, err := http.Get(*pypiURL + pkgName + "/")
59 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
60 http.Error(w, err.Error(), http.StatusBadGateway)
63 if resp.StatusCode != http.StatusOK {
65 log.Println("error", r.RemoteAddr, "refresh", pkgName, "HTTP status:", resp.Status)
66 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
69 body, err := ioutil.ReadAll(resp.Body)
72 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
73 http.Error(w, err.Error(), http.StatusBadGateway)
76 if !mkdirForPkg(w, r, pkgName) {
79 dirPath := filepath.Join(*root, pkgName)
80 for _, lineRaw := range bytes.Split(body, []byte("\n")) {
81 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
82 if len(submatches) == 0 {
86 filename := submatches[2]
87 pkgURL, err := url.Parse(uri)
89 log.Println("error", r.RemoteAddr, "refresh", uri, err)
90 http.Error(w, err.Error(), http.StatusBadGateway)
94 if pkgURL.Fragment == "" {
95 log.Println(r.RemoteAddr, "pypi", filename, "no digest")
96 http.Error(w, "no digest provided", http.StatusBadGateway)
99 digestInfo := strings.Split(pkgURL.Fragment, "=")
100 if len(digestInfo) == 1 {
101 // Ancient non PEP-0503 PyPIs, assume MD5
102 digestInfo = []string{"md5", digestInfo[0]}
103 } else if len(digestInfo) != 2 {
104 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
105 http.Error(w, "invalid digest provided", http.StatusBadGateway)
108 digest, err := hex.DecodeString(digestInfo[1])
110 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
111 http.Error(w, err.Error(), http.StatusBadGateway)
114 hashAlgo := digestInfo[0]
115 var hasherNew func() hash.Hash
122 hasherNew = sha256.New
123 hashSize = sha256.Size
125 hasherNew = sha512.New
126 hashSize = sha512.Size
127 case HashAlgoBLAKE2b256:
128 hasherNew = blake2b256New
129 hashSize = blake2b.Size256
131 log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo)
132 http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
135 if len(digest) != hashSize {
136 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length")
137 http.Error(w, "invalid digest length", http.StatusBadGateway)
142 if pkgURL.Host == "" {
143 uri = pypiURLParsed.ResolveReference(pkgURL).String()
145 uri = pkgURL.String()
148 path := filepath.Join(dirPath, filename)
149 if filename == filenameGet {
151 // Skip heavy remote call, when shutting down
152 http.Error(w, "shutting down", http.StatusInternalServerError)
155 log.Println(r.RemoteAddr, "pypi", filename, "download")
156 resp, err = http.Get(uri)
158 log.Println("error", r.RemoteAddr, "pypi", filename, "download", err)
159 http.Error(w, err.Error(), http.StatusBadGateway)
162 defer resp.Body.Close()
163 if resp.StatusCode != http.StatusOK {
165 "error", r.RemoteAddr,
166 "pypi", filename, "download",
167 "HTTP status:", resp.Status,
169 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
172 hasher := hasherNew()
173 hasherSHA256 := sha256.New()
174 dst, err := TempFile(dirPath)
176 log.Println("error", r.RemoteAddr, "pypi", filename, err)
177 http.Error(w, err.Error(), http.StatusInternalServerError)
180 dstBuf := bufio.NewWriter(dst)
181 wrs := []io.Writer{hasher, dstBuf}
182 if hashAlgo != HashAlgoSHA256 {
183 wrs = append(wrs, hasherSHA256)
185 wr := io.MultiWriter(wrs...)
186 if _, err = io.Copy(wr, resp.Body); err != nil {
187 os.Remove(dst.Name())
189 log.Println("error", r.RemoteAddr, "pypi", filename, err)
190 http.Error(w, err.Error(), http.StatusInternalServerError)
193 if err = dstBuf.Flush(); err != nil {
194 os.Remove(dst.Name())
196 log.Println("error", r.RemoteAddr, "pypi", filename, err)
197 http.Error(w, err.Error(), http.StatusInternalServerError)
200 if bytes.Compare(hasher.Sum(nil), digest) != 0 {
201 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
202 os.Remove(dst.Name())
204 http.Error(w, "digest mismatch", http.StatusBadGateway)
207 if err = dst.Sync(); err != nil {
208 os.Remove(dst.Name())
210 log.Println("error", r.RemoteAddr, "pypi", filename, err)
211 http.Error(w, err.Error(), http.StatusInternalServerError)
214 if err = dst.Close(); err != nil {
215 log.Println("error", r.RemoteAddr, "pypi", filename, err)
216 http.Error(w, err.Error(), http.StatusInternalServerError)
219 if err = os.Rename(dst.Name(), path); err != nil {
220 log.Println("error", r.RemoteAddr, "pypi", filename, err)
221 http.Error(w, err.Error(), http.StatusInternalServerError)
224 if err = DirSync(dirPath); err != nil {
225 log.Println("error", r.RemoteAddr, "pypi", filename, err)
226 http.Error(w, err.Error(), http.StatusInternalServerError)
229 if hashAlgo != HashAlgoSHA256 {
230 hashAlgo = HashAlgoSHA256
231 digest = hasherSHA256.Sum(nil)
232 for _, algo := range knownHashAlgos[1:] {
233 os.Remove(path + "." + algo)
237 if filename == filenameGet || gpgUpdate {
238 if _, err = os.Stat(path); err != nil {
241 resp, err := http.Get(uri + GPGSigExt)
245 if resp.StatusCode != http.StatusOK {
249 sig, err := ioutil.ReadAll(resp.Body)
254 if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
255 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
258 if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
259 log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
260 http.Error(w, err.Error(), http.StatusInternalServerError)
263 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
266 path = path + "." + hashAlgo
267 _, err = os.Stat(path)
271 if !os.IsNotExist(err) {
272 log.Println("error", r.RemoteAddr, "pypi", path, err)
273 http.Error(w, err.Error(), http.StatusInternalServerError)
276 log.Println(r.RemoteAddr, "pypi", filename, "touch")
277 if err = WriteFileSync(dirPath, path, digest); err != nil {
278 log.Println("error", r.RemoteAddr, "pypi", path, err)
279 http.Error(w, err.Error(), http.StatusInternalServerError)